Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

controller,substrate: handle transient Substrate unavailability without permanent Task failure (#463)

aprajitaarora13397f7f27fedb
diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go
index e3daaa9..9d7c3bf 100644
--- a/internal/controller/reconciler.go
+++ b/internal/controller/reconciler.go
@@ -118,6 +118,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wor
// 1. Ensure the Atespace exists in Substrate
if err := r.client.EnsureAtespace(ctx, atespace); err != nil {
+ if substrate.IsTransientError(err) {
+ r.setNotReady(task, "SubstrateUnavailable", err.Error(), now)
+ task.Status.Phase = "Pending"
+ return task, fmt.Errorf("ensuring atespace: %w", err)
+ }
r.setCondition(task, "Ready", "False", "AtespaceCreationFailed", err.Error(), now)
task.Status.Phase = "Failed"
return task, fmt.Errorf("ensuring atespace: %w", err)
@@ -184,6 +189,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wor
_, err := r.client.EnsureActor(ctx, atespace, actorName, templateAtespace, templateName)
if err != nil {
+ if substrate.IsTransientError(err) {
+ r.setNotReady(task, "SubstrateUnavailable", err.Error(), now)
+ task.Status.Phase = "Pending"
+ return task, fmt.Errorf("ensuring actor: %w", err)
+ }
r.setNotReady(task, "ActorCreationFailed", err.Error(), now)
task.Status.Phase = "Failed"
return task, fmt.Errorf("ensuring actor: %w", err)
@@ -207,6 +217,11 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wor
slog.Info("resuming actor on Substrate worker", "actor", actorName)
_, workerIP, err := r.client.ResumeActor(ctx, atespace, actorName)
if err != nil {
+ if substrate.IsTransientError(err) {
+ r.setNotReady(task, "SubstrateUnavailable", err.Error(), now)
+ task.Status.Phase = "Pending"
+ return task, fmt.Errorf("resuming actor: %w", err)
+ }
r.setNotReady(task, "ActorResumeFailed", err.Error(), now)
task.Status.Phase = "Failed"
return task, fmt.Errorf("resuming actor: %w", err)
diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.go
index ba1cd35..1320434 100644
--- a/internal/controller/reconciler_test.go
+++ b/internal/controller/reconciler_test.go
@@ -35,18 +35,21 @@ import (
type mockControlServer struct {
ateapipb.UnimplementedControlServer
- workerIP string
- createdAtespaces []string
- createdActors []string
- resumedActors []string
- suspendedActors []string
- suspendErr error
- deletedActors []string
- actorTemplates map[string]bool
- deletedTemplates []string
- getActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
- crashedActor string
- revertedActors []string
+ workerIP string
+ createdAtespaces []string
+ createdActors []string
+ resumedActors []string
+ suspendedActors []string
+ suspendErr error
+ deletedActors []string
+ actorTemplates map[string]bool
+ deletedTemplates []string
+ getActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
+ createAtespaceErr error
+ createActorErr error
+ resumeActorErr error
+ crashedActor string
+ revertedActors []string
}
// noSecrets is a SecretResolver for tests: it never finds a key and never touches a cluster.
@@ -72,6 +75,9 @@ func (m *mockControlServer) CreateActorTemplate(_ context.Context, req *ateapipb
}
func (m *mockControlServer) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) {
+ if m.createAtespaceErr != nil {
+ return nil, m.createAtespaceErr
+ }
name := ""
if req.Atespace != nil && req.Atespace.Metadata != nil {
name = req.Atespace.Metadata.Name
@@ -81,6 +87,9 @@ func (m *mockControlServer) CreateAtespace(ctx context.Context, req *ateapipb.Cr
}
func (m *mockControlServer) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) {
+ if m.createActorErr != nil {
+ return nil, m.createActorErr
+ }
name := ""
if req.Actor != nil && req.Actor.Metadata != nil {
name = req.Actor.Metadata.Name
@@ -98,6 +107,9 @@ func (m *mockControlServer) CreateActor(ctx context.Context, req *ateapipb.Creat
}
func (m *mockControlServer) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) {
+ if m.resumeActorErr != nil {
+ return nil, m.resumeActorErr
+ }
name := ""
if req.Actor != nil {
name = req.Actor.Name
@@ -680,3 +692,104 @@ func TestEnsureActor_RevertsCrashedActor(t *testing.T) {
t.Errorf("crashed actor: reverted %v, deleted %v; want reverted only", mockSrv.revertedActors, mockSrv.deletedActors)
}
}
+
+func TestTaskReconciler_TransientSubstrateUnavailable(t *testing.T) {
+ ctx := context.Background()
+
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("failed to listen: %v", err)
+ }
+ defer lis.Close()
+
+ mockSrv := &mockControlServer{}
+ grpcServer := grpc.NewServer()
+ ateapipb.RegisterControlServer(grpcServer, mockSrv)
+ go grpcServer.Serve(lis)
+ defer grpcServer.Stop()
+
+ client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ t.Fatalf("failed to create substrate client: %v", err)
+ }
+ defer client.Close()
+
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler.SecretResolver = noSecrets
+ reconciler.WorkspaceReadyTimeout = 50 * time.Millisecond
+
+ // 1. Transient error during EnsureAtespace leaves Task in Phase="Pending" with Reason="SubstrateUnavailable".
+ mockSrv.createAtespaceErr = status.Error(codes.DeadlineExceeded, "substrate control plane timed out")
+ task := &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "transient-atespace"},
+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
+ }
+ reconciled, err := reconciler.Reconcile(ctx, task)
+ if err == nil {
+ t.Fatal("expected error from Reconcile when EnsureAtespace fails, got nil")
+ }
+ if reconciled.GetStatus().GetPhase() != "Pending" {
+ t.Errorf("expected Phase='Pending' on transient EnsureAtespace error, got %q", reconciled.GetStatus().GetPhase())
+ }
+ assertCondition(t, reconciled, "Ready", "False", "SubstrateUnavailable")
+ mockSrv.createAtespaceErr = nil
+
+ // 2. Transient error during EnsureActor leaves Task in Phase="Pending" with Reason="SubstrateUnavailable".
+ mockSrv.createActorErr = status.Error(codes.DeadlineExceeded, "substrate actor creation timed out")
+ task2 := &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "transient-actor"},
+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
+ }
+ reconciled2, err := reconciler.Reconcile(ctx, task2)
+ if err == nil {
+ t.Fatal("expected error from Reconcile when EnsureActor fails, got nil")
+ }
+ if reconciled2.GetStatus().GetPhase() != "Pending" {
+ t.Errorf("expected Phase='Pending' on transient EnsureActor error, got %q", reconciled2.GetStatus().GetPhase())
+ }
+ assertCondition(t, reconciled2, "Ready", "False", "SubstrateUnavailable")
+ mockSrv.createActorErr = nil
+
+ // 3. Transient error during ResumeActor leaves Task in Phase="Pending" with Reason="SubstrateUnavailable",
+ // and a subsequent Reconcile once Substrate recovers transitions the Task to "Running".
+ mockSrv.resumeActorErr = status.Error(codes.DeadlineExceeded, "substrate worker pool temporarily unreachable")
+ task3 := &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "transient-resume"},
+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
+ Status: &v1alpha1.TaskStatus{Phase: "Running"},
+ }
+ reconciled3, err := reconciler.Reconcile(ctx, task3)
+ if err == nil {
+ t.Fatal("expected error from Reconcile when ResumeActor fails, got nil")
+ }
+ if reconciled3.GetStatus().GetPhase() != "Pending" {
+ t.Errorf("expected Phase='Pending' on transient ResumeActor error, got %q", reconciled3.GetStatus().GetPhase())
+ }
+ assertCondition(t, reconciled3, "Ready", "False", "SubstrateUnavailable")
+
+ // Recovery on retry:
+ mockSrv.resumeActorErr = nil
+ reconciled3.Status.Phase = "Running"
+ recovered, err := reconciler.Reconcile(ctx, reconciled3)
+ if err != nil {
+ t.Fatalf("expected Reconcile to succeed after Substrate recovers, got %v", err)
+ }
+ if recovered.GetStatus().GetPhase() != "Running" {
+ t.Errorf("expected Phase='Running' after recovery, got %q", recovered.GetStatus().GetPhase())
+ }
+
+ // 4. Permanent (non-transient) error still marks Task as Phase="Failed".
+ mockSrv.createActorErr = status.Error(codes.InvalidArgument, "invalid actor specification")
+ task4 := &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "permanent-actor-err"},
+ Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
+ }
+ reconciled4, err := reconciler.Reconcile(ctx, task4)
+ if err == nil {
+ t.Fatal("expected error on InvalidArgument, got nil")
+ }
+ if reconciled4.GetStatus().GetPhase() != "Failed" {
+ t.Errorf("expected Phase='Failed' on permanent error, got %q", reconciled4.GetStatus().GetPhase())
+ }
+ mockSrv.createActorErr = nil
+}
diff --git a/internal/server/server.go b/internal/server/server.go
index 2b54a30..b795a2a 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -23,10 +23,12 @@ import (
"github.com/google/ax/internal/lock"
"github.com/google/ax/internal/store"
+ "github.com/google/ax/internal/substrate"
"github.com/google/ax/pkg/apis/v1alpha1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -160,11 +162,14 @@ func (s *Server) CreateTask(ctx context.Context, req *v1alpha1.CreateTaskRequest
}
defer unlock()
- _, err = s.store.GetTask(ctx, atespace, taskName)
+ existing, err := s.store.GetTask(ctx, atespace, taskName)
if err == nil {
- return nil, status.Errorf(codes.FailedPrecondition, "task %s/%s already exists and is immutable", atespace, taskName)
+ if existing.GetStatus().GetPhase() != "Pending" || !proto.Equal(existing.GetSpec(), task.GetSpec()) {
+ return nil, status.Errorf(codes.FailedPrecondition, "task %s/%s already exists and is immutable", atespace, taskName)
+ }
+ task = existing
}
- if !errors.Is(err, store.ErrNotFound) {
+ if err != nil && !errors.Is(err, store.ErrNotFound) {
return nil, status.Errorf(codes.Internal, "checking existing task: %v", err)
}
@@ -185,8 +190,18 @@ func (s *Server) CreateTask(ctx context.Context, req *v1alpha1.CreateTaskRequest
reconciled, err := s.reconciler.Reconcile(ctx, task, workspaces...)
if err != nil {
slog.Error("direct reconcile error on create task", "task", taskName, "error", err)
- task.Status.Phase = "Failed"
+ if reconciled != nil && reconciled.Status != nil {
+ task.Status = reconciled.Status
+ }
+ if substrate.IsTransientError(err) {
+ task.Status.Phase = "Pending"
+ } else if task.Status.Phase != "Pending" {
+ task.Status.Phase = "Failed"
+ }
_ = s.store.UpdateTaskStatus(ctx, atespace, taskName, task.Status)
+ if substrate.IsTransientError(err) {
+ return nil, status.Errorf(codes.Unavailable, "provisioning task on substrate: %v", err)
+ }
return nil, status.Errorf(codes.Internal, "provisioning task on substrate: %v", err)
}
task.Status = reconciled.Status
@@ -287,6 +302,9 @@ func (s *Server) SuspendTask(ctx context.Context, req *v1alpha1.SuspendTaskReque
workspaces := s.fetchWorkspaces(ctx, atespace, task)
reconciled, err := s.reconciler.Reconcile(ctx, task, workspaces...)
if err != nil {
+ if substrate.IsTransientError(err) {
+ return nil, status.Errorf(codes.Unavailable, "suspending task on substrate: %v", err)
+ }
return nil, status.Errorf(codes.Internal, "suspending task on substrate: %v", err)
}
task.Status = reconciled.Status
@@ -335,8 +353,18 @@ func (s *Server) ResumeTask(ctx context.Context, req *v1alpha1.ResumeTaskRequest
workspaces := s.fetchWorkspaces(ctx, atespace, task)
reconciled, err := s.reconciler.Reconcile(ctx, task, workspaces...)
if err != nil {
- task.Status.Phase = "Failed"
+ if reconciled != nil && reconciled.Status != nil {
+ task.Status = reconciled.Status
+ }
+ if substrate.IsTransientError(err) {
+ task.Status.Phase = "Pending"
+ } else if task.Status.Phase != "Pending" {
+ task.Status.Phase = "Failed"
+ }
_ = s.store.UpdateTaskStatus(ctx, atespace, taskName, task.Status)
+ if substrate.IsTransientError(err) {
+ return nil, status.Errorf(codes.Unavailable, "resuming task on substrate: %v", err)
+ }
return nil, status.Errorf(codes.Internal, "resuming task on substrate: %v", err)
}
task.Status = reconciled.Status
@@ -599,4 +627,3 @@ func defaultMetadata(meta *v1alpha1.ObjectMeta, existing func(atespace, name str
}
return meta
}
-
diff --git a/internal/server/server_test.go b/internal/server/server_test.go
index 133c5c7..6c2ffe2 100644
--- a/internal/server/server_test.go
+++ b/internal/server/server_test.go
@@ -31,6 +31,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
)
func TestServerHealthzHTTP(t *testing.T) {
@@ -308,16 +309,29 @@ func TestCreateTask_ValidatesWorkspaceBindings(t *testing.T) {
}
type fakeReconciler struct {
- reconcileCount int
- reconcileErr error
- deleteCount int
- deleteErr error
- onDelete func(ctx context.Context, atespace, taskName string) error
+ reconcileCount int
+ reconcileErr error
+ reconcilePhase string
+ reconcileConditions []*v1alpha1.Condition
+ observedTask *v1alpha1.Task
+ deleteCount int
+ deleteErr error
+ onDelete func(ctx context.Context, atespace, taskName string) error
}
func (f *fakeReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, workspaces ...*v1alpha1.Workspace) (*v1alpha1.Task, error) {
f.reconcileCount++
+ f.observedTask = proto.Clone(task).(*v1alpha1.Task)
if f.reconcileErr != nil {
+ if task.Status == nil {
+ task.Status = &v1alpha1.TaskStatus{}
+ }
+ if f.reconcilePhase != "" {
+ task.Status.Phase = f.reconcilePhase
+ }
+ if f.reconcileConditions != nil {
+ task.Status.Conditions = f.reconcileConditions
+ }
return task, f.reconcileErr
}
task.Status = &v1alpha1.TaskStatus{
@@ -653,3 +667,119 @@ func TestServer_CreateTask_ReconcileErrorMarksFailed(t *testing.T) {
t.Fatalf("expected FailedPrecondition on duplicate create, got %v", err)
}
}
+
+func TestServer_TransientSubstrateUnavailableRecovery(t *testing.T) {
+ st := memory.NewStore()
+ rec := &fakeReconciler{
+ reconcileErr: status.Error(codes.Unavailable, "connection refused"),
+ reconcilePhase: "Pending",
+ reconcileConditions: []*v1alpha1.Condition{
+ {Type: "Ready", Status: "False", Reason: "SubstrateUnavailable"},
+ },
+ }
+ srv := server.NewServer(st, server.Options{Reconciler: rec})
+ ctx := context.Background()
+
+ // 1. CreateTask while Substrate is unavailable records Phase="Pending" (not "Failed")
+ // and returns codes.Unavailable.
+ _, err := srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{
+ Task: &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},
+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},
+ },
+ })
+ if status.Code(err) != codes.Unavailable {
+ t.Fatalf("expected Unavailable from CreateTask when Substrate is unreachable, got %v", err)
+ }
+
+ stored, err := srv.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: "simple-task"})
+ if err != nil {
+ t.Fatalf("GetTask failed: %v", err)
+ }
+ if stored.GetStatus().GetPhase() != "Pending" {
+ t.Errorf("expected stored phase 'Pending' after transient Substrate error, got %q", stored.GetStatus().GetPhase())
+ }
+
+ // 2. Re-applying a Pending task with a different spec is rejected to preserve immutability.
+ _, err = srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{
+ Task: &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},
+ Spec: &v1alpha1.TaskSpec{Image: "debian"},
+ },
+ })
+ if status.Code(err) != codes.FailedPrecondition {
+ t.Fatalf("expected FailedPrecondition when re-applying Pending task with modified spec, got %v", err)
+ }
+
+ // 3. Re-applying a Pending task with omitted status preserves stored status conditions
+ // rather than dropping them before reconciliation.
+ rec.reconcileConditions = nil
+ _, err = srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{
+ Task: &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},
+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},
+ },
+ })
+ if status.Code(err) != codes.Unavailable {
+ t.Fatalf("expected Unavailable on retry while Substrate is still down, got %v", err)
+ }
+ if conds := rec.observedTask.GetStatus().GetConditions(); len(conds) != 1 || conds[0].GetReason() != "SubstrateUnavailable" {
+ t.Errorf("expected stored condition SubstrateUnavailable to be preserved on omitted-status retry, got %v", conds)
+ }
+
+ // 4. Re-applying a Pending task with client-supplied conditions reuses the stored task
+ // rather than overwriting status with client-supplied values.
+ _, err = srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{
+ Task: &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},
+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},
+ Status: &v1alpha1.TaskStatus{
+ Phase: "Running",
+ Conditions: []*v1alpha1.Condition{
+ {Type: "Ready", Status: "True", Reason: "ClientForged"},
+ },
+ },
+ },
+ })
+ if status.Code(err) != codes.Unavailable {
+ t.Fatalf("expected Unavailable on retry while Substrate is still down, got %v", err)
+ }
+ if conds := rec.observedTask.GetStatus().GetConditions(); len(conds) != 1 || conds[0].GetReason() != "SubstrateUnavailable" {
+ t.Errorf("expected client-supplied conditions to be ignored on Pending retry, got %v", conds)
+ }
+
+ // 5. Once Substrate recovers, re-applying CreateTask with the unchanged spec succeeds
+ // for a Pending task and transitions it to Suspended.
+ rec.reconcileErr = nil
+ rec.reconcilePhase = ""
+ created, err := srv.CreateTask(ctx, &v1alpha1.CreateTaskRequest{
+ Task: &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: "simple-task"},
+ Spec: &v1alpha1.TaskSpec{Image: "alpine"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("CreateTask retry after Substrate recovery failed: %v", err)
+ }
+ if created.GetStatus().GetPhase() != "Suspended" {
+ t.Errorf("expected phase 'Suspended' after recovery, got %q", created.GetStatus().GetPhase())
+ }
+ if !created.GetMetadata().GetCreationTimestamp().AsTime().Equal(stored.GetMetadata().GetCreationTimestamp().AsTime()) {
+ t.Errorf("expected CreationTimestamp to be preserved across Pending re-apply")
+ }
+
+ // 6. ResumeTask during a brief Substrate outage keeps the task Pending instead of Failed.
+ rec.reconcileErr = status.Error(codes.Unavailable, "substrate restarting")
+ rec.reconcilePhase = "Pending"
+ _, err = srv.ResumeTask(ctx, &v1alpha1.ResumeTaskRequest{Name: "simple-task"})
+ if status.Code(err) != codes.Unavailable {
+ t.Fatalf("expected Unavailable from ResumeTask when Substrate is unreachable, got %v", err)
+ }
+ storedAfterResumeErr, err := srv.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: "simple-task"})
+ if err != nil {
+ t.Fatalf("GetTask failed: %v", err)
+ }
+ if storedAfterResumeErr.GetStatus().GetPhase() != "Pending" {
+ t.Errorf("expected stored phase 'Pending' after transient ResumeTask error, got %q", storedAfterResumeErr.GetStatus().GetPhase())
+ }
+}
diff --git a/internal/substrate/client.go b/internal/substrate/client.go
index 234016d..be8a85f 100644
--- a/internal/substrate/client.go
+++ b/internal/substrate/client.go
@@ -188,7 +188,10 @@ func (c *Client) EnsureAtespace(ctx context.Context, atespace string) error {
},
},
}
- _, err := c.control.CreateAtespace(ctx, req)
+ err := retryOnCodes(ctx, func() error {
+ _, err := c.control.CreateAtespace(ctx, req)
+ return err
+ }, codes.Unavailable)
if err != nil && status.Code(err) != codes.AlreadyExists {
return fmt.Errorf("creating atespace %q: %w", atespace, err)
}
@@ -203,7 +206,13 @@ func (c *Client) GetActorTemplate(ctx context.Context, atespace, templateName st
Name: templateName,
},
}
- return c.control.GetActorTemplate(ctx, req)
+ var tmpl *ateapipb.ActorTemplate
+ err := retryOnCodes(ctx, func() error {
+ var err error
+ tmpl, err = c.control.GetActorTemplate(ctx, req)
+ return err
+ }, codes.Unavailable)
+ return tmpl, err
}
const (
@@ -236,6 +245,24 @@ var (
actorDeletingWaitInterval = 500 * time.Millisecond
)
+// IsTransientError reports whether err represents a transient Substrate
+// control-plane unavailability or timeout rather than a permanent failure.
+func IsTransientError(err error) bool {
+ if err == nil {
+ return false
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ return true
+ }
+ if st, ok := status.FromError(err); ok {
+ switch st.Code() {
+ case codes.Unavailable, codes.DeadlineExceeded:
+ return true
+ }
+ }
+ return false
+}
+
// retryOnCodes calls fn until it succeeds, returns an error whose gRPC code is not in
// retryable, defaultRetryAttempts is exhausted, or ctx is done. Between attempts it
// waits with jittered exponential backoff starting at defaultRetryInterval and
@@ -344,7 +371,12 @@ func (c *Client) EnsureActorTemplateWithImage(ctx context.Context, baseAtespace,
req := &ateapipb.CreateActorTemplateRequest{
ActorTemplate: tmpl,
}
- created, err := c.control.CreateActorTemplate(ctx, req)
+ var created *ateapipb.ActorTemplate
+ err = retryOnCodes(ctx, func() error {
+ var err error
+ created, err = c.control.CreateActorTemplate(ctx, req)
+ return err
+ }, codes.Unavailable)
if err != nil && status.Code(err) != codes.AlreadyExists {
return nil, fmt.Errorf("creating actor template %s/%s: %w", targetAtespace, targetTemplate, err)
}
@@ -368,7 +400,12 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA
},
},
}
- actor, err := c.control.CreateActor(ctx, req)
+ var actor *ateapipb.Actor
+ err := retryOnCodes(ctx, func() error {
+ var err error
+ actor, err = c.control.CreateActor(ctx, req)
+ return err
+ }, codes.Unavailable)
if err != nil {
if status.Code(err) == codes.AlreadyExists {
existing, getErr := c.GetActor(ctx, atespace, actorName)
@@ -376,12 +413,17 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA
state := existing.GetStatus().GetState()
if state == ateapipb.ActorState_ACTOR_STATE_CRASHED {
// Revert to the last snapshot so the workspace survives the crash.
- reverted, revertErr := c.control.RevertActor(ctx, &ateapipb.RevertActorRequest{
- Actor: &ateapipb.ObjectRef{
- Atespace: atespace,
- Name: actorName,
- },
- })
+ var reverted *ateapipb.RevertActorResponse
+ revertErr := retryOnCodes(ctx, func() error {
+ var err error
+ reverted, err = c.control.RevertActor(ctx, &ateapipb.RevertActorRequest{
+ Actor: &ateapipb.ObjectRef{
+ Atespace: atespace,
+ Name: actorName,
+ },
+ })
+ return err
+ }, codes.Unavailable)
if revertErr == nil {
slog.Warn("existing actor is crashed, reverted to its last snapshot", "actor", actorName)
return reverted.GetActor(), nil
@@ -390,7 +432,11 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA
if err := c.DeleteActor(ctx, atespace, actorName); err != nil {
return nil, err
}
- actor, err = c.control.CreateActor(ctx, req)
+ err = retryOnCodes(ctx, func() error {
+ var err error
+ actor, err = c.control.CreateActor(ctx, req)
+ return err
+ }, codes.Unavailable)
if err == nil {
return actor, nil
}
@@ -435,7 +481,7 @@ func (c *Client) ResumeActor(ctx context.Context, atespace, actorName string) (*
var err error
resp, err = c.control.ResumeActor(ctx, req)
return err
- }, codes.Aborted)
+ }, codes.Aborted, codes.Unavailable)
if err != nil {
return nil, "", fmt.Errorf("resuming actor %s/%s: %w", atespace, actorName, err)
}
@@ -460,7 +506,7 @@ func (c *Client) SuspendActor(ctx context.Context, atespace, actorName string) e
err := retryOnCodes(ctx, func() error {
_, err := c.control.SuspendActor(ctx, req)
return err
- }, codes.Aborted)
+ }, codes.Aborted, codes.Unavailable)
if err != nil {
return fmt.Errorf("suspending actor %s/%s: %w", atespace, actorName, err)
}
@@ -475,7 +521,13 @@ func (c *Client) GetActor(ctx context.Context, atespace, actorName string) (*ate
Name: actorName,
},
}
- return c.control.GetActor(ctx, req)
+ var actor *ateapipb.Actor
+ err := retryOnCodes(ctx, func() error {
+ var err error
+ actor, err = c.control.GetActor(ctx, req)
+ return err
+ }, codes.Unavailable)
+ return actor, err
}
// DeleteActor deletes the specified actor from Substrate and blocks until the actor
@@ -491,7 +543,7 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er
err := retryOnCodes(ctx, func() error {
_, err := c.control.DeleteActor(ctx, req)
return err
- }, codes.Aborted)
+ }, codes.Aborted, codes.Unavailable)
if err != nil {
if status.Code(err) == codes.NotFound {
return nil
@@ -540,7 +592,12 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er
// ListActorTemplates returns all ActorTemplates in the given atespace.
func (c *Client) ListActorTemplates(ctx context.Context, atespace string) ([]*ateapipb.ActorTemplate, error) {
- resp, err := c.control.ListActorTemplates(ctx, &ateapipb.ListActorTemplatesRequest{Atespace: atespace})
+ var resp *ateapipb.ListActorTemplatesResponse
+ err := retryOnCodes(ctx, func() error {
+ var err error
+ resp, err = c.control.ListActorTemplates(ctx, &ateapipb.ListActorTemplatesRequest{Atespace: atespace})
+ return err
+ }, codes.Unavailable)
if err != nil {
return nil, fmt.Errorf("listing actor templates in %s: %w", atespace, err)
}
@@ -561,7 +618,7 @@ func (c *Client) DeleteActorTemplate(ctx context.Context, atespace, templateName
err := retryOnCodes(ctx, func() error {
_, err := c.control.DeleteActorTemplate(ctx, req)
return err
- }, codes.Aborted)
+ }, codes.Aborted, codes.Unavailable)
if err != nil && status.Code(err) != codes.NotFound {
return fmt.Errorf("deleting actor template %s/%s: %w", atespace, templateName, err)
}
diff --git a/internal/substrate/client_test.go b/internal/substrate/client_test.go
index 8e4faff..27b4e8e 100644
--- a/internal/substrate/client_test.go
+++ b/internal/substrate/client_test.go
@@ -413,4 +413,36 @@ func TestRetryOnCodes(t *testing.T) {
t.Errorf("expected context.Canceled, got %v", err)
}
})
+
+ t.Run("retries Unavailable and succeeds", func(t *testing.T) {
+ calls := 0
+ err := retryOnCodes(context.Background(), func() error {
+ calls++
+ if calls < 3 {
+ return status.Error(codes.Unavailable, "ateapi rolling out")
+ }
+ return nil
+ }, codes.Unavailable)
+ if err != nil || calls != 3 {
+ t.Errorf("got err=%v calls=%d, want nil after 3 calls", err, calls)
+ }
+ })
+}
+
+func TestIsTransientError(t *testing.T) {
+ if IsTransientError(nil) {
+ t.Error("expected nil error to be non-transient")
+ }
+ if !IsTransientError(status.Error(codes.Unavailable, "connection refused")) {
+ t.Error("expected codes.Unavailable to be transient")
+ }
+ if !IsTransientError(status.Error(codes.DeadlineExceeded, "rpc deadline exceeded")) {
+ t.Error("expected codes.DeadlineExceeded to be transient")
+ }
+ if !IsTransientError(context.DeadlineExceeded) {
+ t.Error("expected context.DeadlineExceeded to be transient")
+ }
+ if IsTransientError(status.Error(codes.InvalidArgument, "bad spec")) {
+ t.Error("expected codes.InvalidArgument to be non-transient")
+ }
}