Agent Executor (AX) is Google's open agentic orchestration runtime
Go
controller,substrate: handle transient Substrate unavailability without permanent Task failure (#463)
aprajitaarora13397f7f27fedb
diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.goindex e3daaa9..9d7c3bf 100644--- a/internal/controller/reconciler.go+++ b/internal/controller/reconciler.go@@ -118,6 +118,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wor// 1. Ensure the Atespace exists in Substrateif err := r.client.EnsureAtespace(ctx, atespace); err != nil {+ if substrate.IsTransientError(err) {+ r.setNotReady(task, "SubstrateUnavailable", err.Error(), now)+ task.Status.Phase = "Pending"+ return task, fmt.Errorf("ensuring atespace: %w", err)+ }r.setCondition(task, "Ready", "False", "AtespaceCreationFailed", err.Error(), now)task.Status.Phase = "Failed"return task, fmt.Errorf("ensuring atespace: %w", err)@@ -184,6 +189,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wor_, err := r.client.EnsureActor(ctx, atespace, actorName, templateAtespace, templateName)if err != nil {+ if substrate.IsTransientError(err) {+ r.setNotReady(task, "SubstrateUnavailable", err.Error(), now)+ task.Status.Phase = "Pending"+ return task, fmt.Errorf("ensuring actor: %w", err)+ }r.setNotReady(task, "ActorCreationFailed", err.Error(), now)task.Status.Phase = "Failed"return task, fmt.Errorf("ensuring actor: %w", err)@@ -207,6 +217,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, worslog.Info("resuming actor on Substrate worker", "actor", actorName)_, workerIP, err := r.client.ResumeActor(ctx, atespace, actorName)if err != nil {+ if substrate.IsTransientError(err) {+ r.setNotReady(task, "SubstrateUnavailable", err.Error(), now)+ task.Status.Phase = "Pending"+ return task, fmt.Errorf("resuming actor: %w", err)+ }r.setNotReady(task, "ActorResumeFailed", err.Error(), now)task.Status.Phase = "Failed"return task, fmt.Errorf("resuming actor: %w", err)diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.goindex ba1cd35..1320434 100644--- a/internal/controller/reconciler_test.go+++ b/internal/controller/reconciler_test.go@@ -35,18 +35,21 @@ import (type mockControlServer struct {ateapipb.UnimplementedControlServer- workerIP string- createdAtespaces []string- createdActors []string- resumedActors []string- suspendedActors []string- suspendErr error- deletedActors []string- actorTemplates map[string]bool- deletedTemplates []string- getActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)- crashedActor string- revertedActors []string+ workerIP string+ createdAtespaces []string+ createdActors []string+ resumedActors []string+ suspendedActors []string+ suspendErr error+ deletedActors []string+ actorTemplates map[string]bool+ deletedTemplates []string+ getActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)+ createAtespaceErr error+ createActorErr error+ resumeActorErr error+ crashedActor string+ revertedActors []string}// noSecrets is a SecretResolver for tests: it never finds a key and never touches a cluster.@@ -72,6 +75,9 @@ func (m *mockControlServer) CreateActorTemplate(_ context.Context, req *ateapipb}func (m *mockControlServer) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) {+ if m.createAtespaceErr != nil {+ return nil, m.createAtespaceErr+ }name := ""if req.Atespace != nil && req.Atespace.Metadata != nil {name = req.Atespace.Metadata.Name@@ -81,6 +87,9 @@ func (m *mockControlServer) CreateAtespace(ctx context.Context, req *ateapipb.Cr}func (m *mockControlServer) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) {+ if m.createActorErr != nil {+ return nil, m.createActorErr+ }name := ""if req.Actor != nil && req.Actor.Metadata != nil {name = req.Actor.Metadata.Name@@ -98,6 +107,9 @@ func (m *mockControlServer) CreateActor(ctx context.Context, req *ateapipb.Creat}func (m *mockControlServer) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) {+ if m.resumeActorErr != nil {+ return nil, m.resumeActorErr+ }name := ""if req.Actor != nil {name = req.Actor.Name@@ -680,3 +692,104 @@ func TestEnsureActor_RevertsCrashedActor(t *testing.T) {t.Errorf("crashed actor: reverted %v, deleted %v; want reverted only", mockSrv.revertedActors, mockSrv.deletedActors)}}++func TestTaskReconciler_TransientSubstrateUnavailable(t *testing.T) {+ ctx := context.Background()++ lis, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ defer lis.Close()++ mockSrv := &mockControlServer{}+ grpcServer := grpc.NewServer()+ ateapipb.RegisterControlServer(grpcServer, mockSrv)+ go grpcServer.Serve(lis)+ defer grpcServer.Stop()++ client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))+ if err != nil {+ t.Fatalf("failed to create substrate client: %v", err)+ }+ defer client.Close()++ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler.SecretResolver = noSecrets+ reconciler.WorkspaceReadyTimeout = 50 * time.Millisecond++ // 1. Transient error during EnsureAtespace leaves Task in Phase="Pending" with Reason="SubstrateUnavailable".+ mockSrv.createAtespaceErr = status.Error(codes.DeadlineExceeded, "substrate control plane timed out")+ task := &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "transient-atespace"},+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},+ }+ reconciled, err := reconciler.Reconcile(ctx, task)+ if err == nil {+ t.Fatal("expected error from Reconcile when EnsureAtespace fails, got nil")+ }+ if reconciled.GetStatus().GetPhase() != "Pending" {+ t.Errorf("expected Phase='Pending' on transient EnsureAtespace error, got %q", reconciled.GetStatus().GetPhase())+ }+ assertCondition(t, reconciled, "Ready", "False", "SubstrateUnavailable")+ mockSrv.createAtespaceErr = nil++ // 2. Transient error during EnsureActor leaves Task in Phase="Pending" with Reason="SubstrateUnavailable".+ mockSrv.createActorErr = status.Error(codes.DeadlineExceeded, "substrate actor creation timed out")+ task2 := &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "transient-actor"},+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},+ }+ reconciled2, err := reconciler.Reconcile(ctx, task2)+ if err == nil {+ t.Fatal("expected error from Reconcile when EnsureActor fails, got nil")+ }+ if reconciled2.GetStatus().GetPhase() != "Pending" {+ t.Errorf("expected Phase='Pending' on transient EnsureActor error, got %q", reconciled2.GetStatus().GetPhase())+ }+ assertCondition(t, reconciled2, "Ready", "False", "SubstrateUnavailable")+ mockSrv.createActorErr = nil++ // 3. Transient error during ResumeActor leaves Task in Phase="Pending" with Reason="SubstrateUnavailable",+ // and a subsequent Reconcile once Substrate recovers transitions the Task to "Running".+ mockSrv.resumeActorErr = status.Error(codes.DeadlineExceeded, "substrate worker pool temporarily unreachable")+ task3 := &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "transient-resume"},+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},+ Status: &v1alpha1.TaskStatus{Phase: "Running"},+ }+ reconciled3, err := reconciler.Reconcile(ctx, task3)+ if err == nil {+ t.Fatal("expected error from Reconcile when ResumeActor fails, got nil")+ }+ if reconciled3.GetStatus().GetPhase() != "Pending" {+ t.Errorf("expected Phase='Pending' on transient ResumeActor error, got %q", reconciled3.GetStatus().GetPhase())+ }+ assertCondition(t, reconciled3, "Ready", "False", "SubstrateUnavailable")++ // Recovery on retry:+ mockSrv.resumeActorErr = nil+ reconciled3.Status.Phase = "Running"+ recovered, err := reconciler.Reconcile(ctx, reconciled3)+ if err != nil {+ t.Fatalf("expected Reconcile to succeed after Substrate recovers, got %v", err)+ }+ if recovered.GetStatus().GetPhase() != "Running" {+ t.Errorf("expected Phase='Running' after recovery, got %q", recovered.GetStatus().GetPhase())+ }++ // 4. Permanent (non-transient) error still marks Task as Phase="Failed".+ mockSrv.createActorErr = status.Error(codes.InvalidArgument, "invalid actor specification")+ task4 := &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "permanent-actor-err"},+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},+ }+ reconciled4, err := reconciler.Reconcile(ctx, task4)+ if err == nil {+ t.Fatal("expected error on InvalidArgument, got nil")+ }+ if reconciled4.GetStatus().GetPhase() != "Failed" {+ t.Errorf("expected Phase='Failed' on permanent error, got %q", reconciled4.GetStatus().GetPhase())+ }+ mockSrv.createActorErr = nil+}diff --git a/internal/server/server.go b/internal/server/server.goindex 2b54a30..b795a2a 100644--- a/internal/server/server.go+++ b/internal/server/server.go@@ -23,10 +23,12 @@ import ("github.com/google/ax/internal/lock""github.com/google/ax/internal/store"+ "github.com/google/ax/internal/substrate""github.com/google/ax/pkg/apis/v1alpha1""google.golang.org/grpc""google.golang.org/grpc/codes""google.golang.org/grpc/status"+ "google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/timestamppb")@@ -160,11 +162,14 @@ func (s *Server) CreateTask(ctx context.Context, req *v1alpha1.CreateTaskRequest}defer unlock()- _, err = s.store.GetTask(ctx, atespace, taskName)+ existing, err := s.store.GetTask(ctx, atespace, taskName)if err == nil {- return nil, status.Errorf(codes.FailedPrecondition, "task %s/%s already exists and is immutable", atespace, taskName)+ if existing.GetStatus().GetPhase() != "Pending" || !proto.Equal(existing.GetSpec(), task.GetSpec()) {+ return nil, status.Errorf(codes.FailedPrecondition, "task %s/%s already exists and is immutable", atespace, taskName)+ }+ task = existing}- if !errors.Is(err, store.ErrNotFound) {+ if err != nil && !errors.Is(err, store.ErrNotFound) {return nil, status.Errorf(codes.Internal, "checking existing task: %v", err)}@@ -185,8 +190,18 @@ func (s *Server) CreateTask(ctx context.Context, req *v1alpha1.CreateTaskRequestreconciled, err := s.reconciler.Reconcile(ctx, task, workspaces...)if err != nil {slog.Error("direct reconcile error on create task", "task", taskName, "error", err)- task.Status.Phase = "Failed"+ if reconciled != nil && reconciled.Status != nil {+ task.Status = reconciled.Status+ }+ if substrate.IsTransientError(err) {+ task.Status.Phase = "Pending"+ } else if task.Status.Phase != "Pending" {+ task.Status.Phase = "Failed"+ }_ = s.store.UpdateTaskStatus(ctx, atespace, taskName, task.Status)+ if substrate.IsTransientError(err) {+ return nil, status.Errorf(codes.Unavailable, "provisioning task on substrate: %v", err)+ }return nil, status.Errorf(codes.Internal, "provisioning task on substrate: %v", err)}task.Status = reconciled.Status@@ -287,6 +302,9 @@ func (s *Server) SuspendTask(ctx context.Context, req *v1alpha1.SuspendTaskRequeworkspaces := s.fetchWorkspaces(ctx, atespace, task)reconciled, err := s.reconciler.Reconcile(ctx, task, workspaces...)if err != nil {+ if substrate.IsTransientError(err) {+ return nil, status.Errorf(codes.Unavailable, "suspending task on substrate: %v", err)+ }return nil, status.Errorf(codes.Internal, "suspending task on substrate: %v", err)}task.Status = reconciled.Status@@ -335,8 +353,18 @@ func (s *Server) ResumeTask(ctx context.Context, req *v1alpha1.ResumeTaskRequestworkspaces := s.fetchWorkspaces(ctx, atespace, task)reconciled, err := s.reconciler.Reconcile(ctx, task, workspaces...)if err != nil {- task.Status.Phase = "Failed"+ if reconciled != nil && reconciled.Status != nil {+ task.Status = reconciled.Status+ }+ if substrate.IsTransientError(err) {+ task.Status.Phase = "Pending"+ } else if task.Status.Phase != "Pending" {+ task.Status.Phase = "Failed"+ }_ = s.store.UpdateTaskStatus(ctx, atespace, taskName, task.Status)+ if substrate.IsTransientError(err) {+ return nil, status.Errorf(codes.Unavailable, "resuming task on substrate: %v", err)+ }return nil, status.Errorf(codes.Internal, "resuming task on substrate: %v", err)}task.Status = reconciled.Status@@ -599,4 +627,3 @@ func defaultMetadata(meta *v1alpha1.ObjectMeta, existing func(atespace, name str}return meta}-diff --git a/internal/server/server_test.go b/internal/server/server_test.goindex 133c5c7..6c2ffe2 100644--- a/internal/server/server_test.go+++ b/internal/server/server_test.go@@ -31,6 +31,7 @@ import ("google.golang.org/grpc/codes""google.golang.org/grpc/credentials/insecure""google.golang.org/grpc/status"+ "google.golang.org/protobuf/proto")func TestServerHealthzHTTP(t *testing.T) {@@ -308,16 +309,29 @@ func TestCreateTask_ValidatesWorkspaceBindings(t *testing.T) {}type fakeReconciler struct {- reconcileCount int- reconcileErr error- deleteCount int- deleteErr error- onDelete func(ctx context.Context, atespace, taskName string) error+ reconcileCount int+ reconcileErr error+ reconcilePhase string+ reconcileConditions []*v1alpha1.Condition+ observedTask *v1alpha1.Task+ deleteCount int+ deleteErr error+ onDelete func(ctx context.Context, atespace, taskName string) error}func (f *fakeReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, workspaces ...*v1alpha1.Workspace) (*v1alpha1.Task, error) {f.reconcileCount+++ f.observedTask = proto.Clone(task).(*v1alpha1.Task)if f.reconcileErr != nil {+ if task.Status == nil {+ task.Status = &v1alpha1.TaskStatus{}+ }+ if f.reconcilePhase != "" {+ task.Status.Phase = f.reconcilePhase+ }+ if f.reconcileConditions != nil {+ task.Status.Conditions = f.reconcileConditions+ }return task, f.reconcileErr}task.Status = &v1alpha1.TaskStatus{@@ -653,3 +667,119 @@ func TestServer_CreateTask_ReconcileErrorMarksFailed(t *testing.T) {t.Fatalf("expected FailedPrecondition on duplicate create, got %v", err)}}++func TestServer_TransientSubstrateUnavailableRecovery(t *testing.T) {+ st := memory.NewStore()+ rec := &fakeReconciler{+ reconcileErr: status.Error(codes.Unavailable, "connection refused"),+ reconcilePhase: "Pending",+ reconcileConditions: []*v1alpha1.Condition{+ {Type: "Ready", Status: "False", Reason: "SubstrateUnavailable"},+ },+ }+ srv := server.NewServer(st, server.Options{Reconciler: rec})+ ctx := context.Background()++ // 1. CreateTask while Substrate is unavailable records Phase="Pending" (not "Failed")+ // and returns codes.Unavailable.+ _, err := srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{+ Task: &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},+ },+ })+ if status.Code(err) != codes.Unavailable {+ t.Fatalf("expected Unavailable from CreateTask when Substrate is unreachable, got %v", err)+ }++ stored, err := srv.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: "simple-task"})+ if err != nil {+ t.Fatalf("GetTask failed: %v", err)+ }+ if stored.GetStatus().GetPhase() != "Pending" {+ t.Errorf("expected stored phase 'Pending' after transient Substrate error, got %q", stored.GetStatus().GetPhase())+ }++ // 2. Re-applying a Pending task with a different spec is rejected to preserve immutability.+ _, err = srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{+ Task: &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},+ Spec: &v1alpha1.TaskSpec{Image: "debian"},+ },+ })+ if status.Code(err) != codes.FailedPrecondition {+ t.Fatalf("expected FailedPrecondition when re-applying Pending task with modified spec, got %v", err)+ }++ // 3. Re-applying a Pending task with omitted status preserves stored status conditions+ // rather than dropping them before reconciliation.+ rec.reconcileConditions = nil+ _, err = srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{+ Task: &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},+ },+ })+ if status.Code(err) != codes.Unavailable {+ t.Fatalf("expected Unavailable on retry while Substrate is still down, got %v", err)+ }+ if conds := rec.observedTask.GetStatus().GetConditions(); len(conds) != 1 || conds[0].GetReason() != "SubstrateUnavailable" {+ t.Errorf("expected stored condition SubstrateUnavailable to be preserved on omitted-status retry, got %v", conds)+ }++ // 4. Re-applying a Pending task with client-supplied conditions reuses the stored task+ // rather than overwriting status with client-supplied values.+ _, err = srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{+ Task: &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},+ Status: &v1alpha1.TaskStatus{+ Phase: "Running",+ Conditions: []*v1alpha1.Condition{+ {Type: "Ready", Status: "True", Reason: "ClientForged"},+ },+ },+ },+ })+ if status.Code(err) != codes.Unavailable {+ t.Fatalf("expected Unavailable on retry while Substrate is still down, got %v", err)+ }+ if conds := rec.observedTask.GetStatus().GetConditions(); len(conds) != 1 || conds[0].GetReason() != "SubstrateUnavailable" {+ t.Errorf("expected client-supplied conditions to be ignored on Pending retry, got %v", conds)+ }++ // 5. Once Substrate recovers, re-applying CreateTask with the unchanged spec succeeds+ // for a Pending task and transitions it to Suspended.+ rec.reconcileErr = nil+ rec.reconcilePhase = ""+ created, err := srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{+ Task: &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},+ },+ })+ if err != nil {+ t.Fatalf("CreateTask retry after Substrate recovery failed: %v", err)+ }+ if created.GetStatus().GetPhase() != "Suspended" {+ t.Errorf("expected phase 'Suspended' after recovery, got %q", created.GetStatus().GetPhase())+ }+ if !created.GetMetadata().GetCreationTimestamp().AsTime().Equal(stored.GetMetadata().GetCreationTimestamp().AsTime()) {+ t.Errorf("expected CreationTimestamp to be preserved across Pending re-apply")+ }++ // 6. ResumeTask during a brief Substrate outage keeps the task Pending instead of Failed.+ rec.reconcileErr = status.Error(codes.Unavailable, "substrate restarting")+ rec.reconcilePhase = "Pending"+ _, err = srv.ResumeTask(ctx, &v1alpha1.ResumeTaskRequest{Name: "simple-task"})+ if status.Code(err) != codes.Unavailable {+ t.Fatalf("expected Unavailable from ResumeTask when Substrate is unreachable, got %v", err)+ }+ storedAfterResumeErr, err := srv.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: "simple-task"})+ if err != nil {+ t.Fatalf("GetTask failed: %v", err)+ }+ if storedAfterResumeErr.GetStatus().GetPhase() != "Pending" {+ t.Errorf("expected stored phase 'Pending' after transient ResumeTask error, got %q", storedAfterResumeErr.GetStatus().GetPhase())+ }+}diff --git a/internal/substrate/client.go b/internal/substrate/client.goindex 234016d..be8a85f 100644--- a/internal/substrate/client.go+++ b/internal/substrate/client.go@@ -188,7 +188,10 @@ func (c *Client) EnsureAtespace(ctx context.Context, atespace string) error {},},}- _, err := c.control.CreateAtespace(ctx, req)+ err := retryOnCodes(ctx, func() error {+ _, err := c.control.CreateAtespace(ctx, req)+ return err+ }, codes.Unavailable)if err != nil && status.Code(err) != codes.AlreadyExists {return fmt.Errorf("creating atespace %q: %w", atespace, err)}@@ -203,7 +206,13 @@ func (c *Client) GetActorTemplate(ctx context.Context, atespace, templateName stName: templateName,},}- return c.control.GetActorTemplate(ctx, req)+ var tmpl *ateapipb.ActorTemplate+ err := retryOnCodes(ctx, func() error {+ var err error+ tmpl, err = c.control.GetActorTemplate(ctx, req)+ return err+ }, codes.Unavailable)+ return tmpl, err}const (@@ -236,6 +245,24 @@ var (actorDeletingWaitInterval = 500 * time.Millisecond)+// IsTransientError reports whether err represents a transient Substrate+// control-plane unavailability or timeout rather than a permanent failure.+func IsTransientError(err error) bool {+ if err == nil {+ return false+ }+ if errors.Is(err, context.DeadlineExceeded) {+ return true+ }+ if st, ok := status.FromError(err); ok {+ switch st.Code() {+ case codes.Unavailable, codes.DeadlineExceeded:+ return true+ }+ }+ return false+}+// retryOnCodes calls fn until it succeeds, returns an error whose gRPC code is not in// retryable, defaultRetryAttempts is exhausted, or ctx is done. Between attempts it// waits with jittered exponential backoff starting at defaultRetryInterval and@@ -344,7 +371,12 @@ func (c *Client) EnsureActorTemplateWithImage(ctx context.Context, baseAtespace,req := &ateapipb.CreateActorTemplateRequest{ActorTemplate: tmpl,}- created, err := c.control.CreateActorTemplate(ctx, req)+ var created *ateapipb.ActorTemplate+ err = retryOnCodes(ctx, func() error {+ var err error+ created, err = c.control.CreateActorTemplate(ctx, req)+ return err+ }, codes.Unavailable)if err != nil && status.Code(err) != codes.AlreadyExists {return nil, fmt.Errorf("creating actor template %s/%s: %w", targetAtespace, targetTemplate, err)}@@ -368,7 +400,12 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA},},}- actor, err := c.control.CreateActor(ctx, req)+ var actor *ateapipb.Actor+ err := retryOnCodes(ctx, func() error {+ var err error+ actor, err = c.control.CreateActor(ctx, req)+ return err+ }, codes.Unavailable)if err != nil {if status.Code(err) == codes.AlreadyExists {existing, getErr := c.GetActor(ctx, atespace, actorName)@@ -376,12 +413,17 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateAstate := existing.GetStatus().GetState()if state == ateapipb.ActorState_ACTOR_STATE_CRASHED {// Revert to the last snapshot so the workspace survives the crash.- reverted, revertErr := c.control.RevertActor(ctx, &ateapipb.RevertActorRequest{- Actor: &ateapipb.ObjectRef{- Atespace: atespace,- Name: actorName,- },- })+ var reverted *ateapipb.RevertActorResponse+ revertErr := retryOnCodes(ctx, func() error {+ var err error+ reverted, err = c.control.RevertActor(ctx, &ateapipb.RevertActorRequest{+ Actor: &ateapipb.ObjectRef{+ Atespace: atespace,+ Name: actorName,+ },+ })+ return err+ }, codes.Unavailable)if revertErr == nil {slog.Warn("existing actor is crashed, reverted to its last snapshot", "actor", actorName)return reverted.GetActor(), nil@@ -390,7 +432,11 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateAif err := c.DeleteActor(ctx, atespace, actorName); err != nil {return nil, err}- actor, err = c.control.CreateActor(ctx, req)+ err = retryOnCodes(ctx, func() error {+ var err error+ actor, err = c.control.CreateActor(ctx, req)+ return err+ }, codes.Unavailable)if err == nil {return actor, nil}@@ -435,7 +481,7 @@ func (c *Client) ResumeActor(ctx context.Context, atespace, actorName string) (*var err errorresp, err = c.control.ResumeActor(ctx, req)return err- }, codes.Aborted)+ }, codes.Aborted, codes.Unavailable)if err != nil {return nil, "", fmt.Errorf("resuming actor %s/%s: %w", atespace, actorName, err)}@@ -460,7 +506,7 @@ func (c *Client) SuspendActor(ctx context.Context, atespace, actorName string) eerr := retryOnCodes(ctx, func() error {_, err := c.control.SuspendActor(ctx, req)return err- }, codes.Aborted)+ }, codes.Aborted, codes.Unavailable)if err != nil {return fmt.Errorf("suspending actor %s/%s: %w", atespace, actorName, err)}@@ -475,7 +521,13 @@ func (c *Client) GetActor(ctx context.Context, atespace, actorName string) (*ateName: actorName,},}- return c.control.GetActor(ctx, req)+ var actor *ateapipb.Actor+ err := retryOnCodes(ctx, func() error {+ var err error+ actor, err = c.control.GetActor(ctx, req)+ return err+ }, codes.Unavailable)+ return actor, err}// DeleteActor deletes the specified actor from Substrate and blocks until the actor@@ -491,7 +543,7 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) ererr := retryOnCodes(ctx, func() error {_, err := c.control.DeleteActor(ctx, req)return err- }, codes.Aborted)+ }, codes.Aborted, codes.Unavailable)if err != nil {if status.Code(err) == codes.NotFound {return nil@@ -540,7 +592,12 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er// ListActorTemplates returns all ActorTemplates in the given atespace.func (c *Client) ListActorTemplates(ctx context.Context, atespace string) ([]*ateapipb.ActorTemplate, error) {- resp, err := c.control.ListActorTemplates(ctx, &ateapipb.ListActorTemplatesRequest{Atespace: atespace})+ var resp *ateapipb.ListActorTemplatesResponse+ err := retryOnCodes(ctx, func() error {+ var err error+ resp, err = c.control.ListActorTemplates(ctx, &ateapipb.ListActorTemplatesRequest{Atespace: atespace})+ return err+ }, codes.Unavailable)if err != nil {return nil, fmt.Errorf("listing actor templates in %s: %w", atespace, err)}@@ -561,7 +618,7 @@ func (c *Client) DeleteActorTemplate(ctx context.Context, atespace, templateNameerr := retryOnCodes(ctx, func() error {_, err := c.control.DeleteActorTemplate(ctx, req)return err- }, codes.Aborted)+ }, codes.Aborted, codes.Unavailable)if err != nil && status.Code(err) != codes.NotFound {return fmt.Errorf("deleting actor template %s/%s: %w", atespace, templateName, err)}diff --git a/internal/substrate/client_test.go b/internal/substrate/client_test.goindex 8e4faff..27b4e8e 100644--- a/internal/substrate/client_test.go+++ b/internal/substrate/client_test.go@@ -413,4 +413,36 @@ func TestRetryOnCodes(t *testing.T) {t.Errorf("expected context.Canceled, got %v", err)}})++ t.Run("retries Unavailable and succeeds", func(t *testing.T) {+ calls := 0+ err := retryOnCodes(context.Background(), func() error {+ calls+++ if calls < 3 {+ return status.Error(codes.Unavailable, "ateapi rolling out")+ }+ return nil+ }, codes.Unavailable)+ if err != nil || calls != 3 {+ t.Errorf("got err=%v calls=%d, want nil after 3 calls", err, calls)+ }+ })+}++func TestIsTransientError(t *testing.T) {+ if IsTransientError(nil) {+ t.Error("expected nil error to be non-transient")+ }+ if !IsTransientError(status.Error(codes.Unavailable, "connection refused")) {+ t.Error("expected codes.Unavailable to be transient")+ }+ if !IsTransientError(status.Error(codes.DeadlineExceeded, "rpc deadline exceeded")) {+ t.Error("expected codes.DeadlineExceeded to be transient")+ }+ if !IsTransientError(context.DeadlineExceeded) {+ t.Error("expected context.DeadlineExceeded to be transient")+ }+ if IsTransientError(status.Error(codes.InvalidArgument, "bad spec")) {+ t.Error("expected codes.InvalidArgument to be non-transient")+ }}