Agent Executor (AX) is Google's open agentic orchestration runtime
Go
fix(substrate): revert crashed actors instead of recreating them (#432)
Alex Jones9edb7b1b0ca5
diff --git a/go.mod b/go.modindex 3f8918d..89b69cf 100644--- a/go.mod+++ b/go.mod@@ -4,7 +4,7 @@ go 1.27.1require (github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170- github.com/agent-substrate/substrate v0.0.0-20260911232748-672533541dbf+ github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8github.com/redis/go-redis/v9 v9.22.0google.golang.org/grpc v1.83.2google.golang.org/protobuf v1.36.12diff --git a/go.sum b/go.sumindex 5287948..27abe18 100644--- a/go.sum+++ b/go.sum@@ -1,7 +1,7 @@github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170 h1:8AxhEx3BL7J8c+zqnYq5S3aH0J/4z85/JYWoo1dr6xY=github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170/go.mod h1:RZ4ACmDqWQx7fgX9b1EaFJskeYTZh1Y7uZinDtp7U2o=-github.com/agent-substrate/substrate v0.0.0-20260911232748-672533541dbf h1:FC9Ed75cCczI6unzgMe1Ss3J0mzX2GOiASSHBQnIl/E=-github.com/agent-substrate/substrate v0.0.0-20260911232748-672533541dbf/go.mod h1:WBkGfDCbVFbtJTEMpIIbq59yEgko1L4WHJIEVGBdems=+github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8 h1:C39j2y88zDvg0VVrIfJqYejCjeHI25hugGubMeee74w=+github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8/go.mod h1:pWnhZpFkHq9VjF2OxsqEbryLnnUYmohhC62bFoindCM=github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.goindex 7143109..159184c 100644--- a/internal/controller/reconciler_test.go+++ b/internal/controller/reconciler_test.go@@ -41,6 +41,8 @@ type mockControlServer struct {deletedActors []stringactorTemplates map[string]booldeletedTemplates []string+ crashedActor string+ revertedActors []string}// noSecrets is a SecretResolver for tests: it never finds a key and never touches a cluster.@@ -79,6 +81,9 @@ func (m *mockControlServer) CreateActor(ctx context.Context, req *ateapipb.Creatif req.Actor != nil && req.Actor.Metadata != nil {name = req.Actor.Metadata.Name}+ if name == m.crashedActor {+ return nil, status.Error(codes.AlreadyExists, "actor exists")+ }m.createdActors = append(m.createdActors, name)return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Name: name},@@ -123,6 +128,22 @@ func (m *mockControlServer) SuspendActor(ctx context.Context, req *ateapipb.Susp}+func (m *mockControlServer) GetActor(_ context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) {+ return &ateapipb.Actor{+ Metadata: &ateapipb.ResourceMetadata{Name: req.GetActor().GetName()},+ Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_CRASHED},+ }, nil+}++func (m *mockControlServer) RevertActor(_ context.Context, req *ateapipb.RevertActorRequest) (*ateapipb.RevertActorResponse, error) {+ name := req.GetActor().GetName()+ m.revertedActors = append(m.revertedActors, name)+ return &ateapipb.RevertActorResponse{Actor: &ateapipb.Actor{+ Metadata: &ateapipb.ResourceMetadata{Name: name},+ Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},+ }}, nil+}+func (m *mockControlServer) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) {name := req.GetActor().GetName()m.deletedActors = append(m.deletedActors, name)@@ -459,3 +480,34 @@ func TestReconcileDelete_RemovesActorAndTemplates(t *testing.T) {}}}++func TestEnsureActor_RevertsCrashedActor(t *testing.T) {+ lis, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ defer lis.Close()++ mockSrv := &mockControlServer{crashedActor: "job"}+ grpcServer := grpc.NewServer()+ ateapipb.RegisterControlServer(grpcServer, mockSrv)+ go grpcServer.Serve(lis)+ defer grpcServer.Stop()++ client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))+ if err != nil {+ t.Fatalf("failed to create substrate client: %v", err)+ }+ defer client.Close()++ actor, err := client.EnsureActor(context.Background(), "default", "job", "default", "job-tmpl-0a1b2c3d")+ if err != nil {+ t.Fatalf("EnsureActor failed: %v", err)+ }+ if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED {+ t.Errorf("actor state = %v, want SUSPENDED", got)+ }+ if len(mockSrv.revertedActors) != 1 || len(mockSrv.deletedActors) != 0 {+ t.Errorf("crashed actor: reverted %v, deleted %v; want reverted only", mockSrv.revertedActors, mockSrv.deletedActors)+ }+}diff --git a/internal/substrate/client.go b/internal/substrate/client.goindex b0273e8..a94a488 100644--- a/internal/substrate/client.go+++ b/internal/substrate/client.go@@ -329,7 +329,18 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateAif getErr == nil && existing != nil {state := existing.GetStatus().GetState()if state == ateapipb.ActorState_ACTOR_STATE_CRASHED {- slog.Warn("existing actor is crashed, deleting and recreating", "actor", actorName)+ // Revert to the last snapshot so the workspace survives the crash.+ reverted, revertErr := c.control.RevertActor(ctx, &ateapipb.RevertActorRequest{+ Actor: &ateapipb.ObjectRef{+ Atespace: atespace,+ Name: actorName,+ },+ })+ if revertErr == nil {+ slog.Warn("existing actor is crashed, reverted to its last snapshot", "actor", actorName)+ return reverted.GetActor(), nil+ }+ slog.Warn("existing actor is crashed and could not be reverted, deleting and recreating", "actor", actorName, "error", revertErr)_, _ = c.control.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: &ateapipb.ObjectRef{Atespace: atespace,