Agent Executor (AX) is Google's open agentic orchestration runtime
Go
fix(controller): skip reconcile for tasks that are being deleted (#417)
Mesut Oezdil2fbce2edcd85
diff --git a/internal/controller/worker.go b/internal/controller/worker.goindex 8af08e5..3f36f41 100644--- a/internal/controller/worker.go+++ b/internal/controller/worker.go@@ -126,6 +126,12 @@ func (w *Worker) processEvent(ctx context.Context, ev store.TaskEvent) error {}return fmt.Errorf("fetching task %s/%s: %w", ev.Atespace, ev.Name, err)}+ // A pending delete event owns this task now; reconciling would resume an actor+ // that is about to be torn down and overwrite the Terminating phase.+ if task.GetStatus().GetPhase() == v1alpha1.PhaseTerminating {+ slog.Info("task is terminating, skipping reconcile", "atespace", ev.Atespace, "name", ev.Name)+ return nil+ }// Resolve every bound workspace. A missing one is skipped so the task still// runs; the runner creates an empty directory at its path.diff --git a/internal/controller/worker_test.go b/internal/controller/worker_test.goindex 49b066c..f152bef 100644--- a/internal/controller/worker_test.go+++ b/internal/controller/worker_test.go@@ -173,3 +173,61 @@ func TestWorkerDeletion(t *testing.T) {t.Errorf("expected template deleted, got %v", mockSrv.deletedTemplates)}}++func TestWorkerSkipsReconcileOfTerminatingTask(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ lis, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ defer lis.Close()++ mockSrv := &mockControlServer{}+ grpcServer := grpc.NewServer()+ ateapipb.RegisterControlServer(grpcServer, mockSrv)+ go grpcServer.Serve(lis)+ defer grpcServer.Stop()++ subClient, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))+ if err != nil {+ t.Fatalf("failed to create substrate client: %v", err)+ }+ defer subClient.Close()++ reconciler := controller.NewTaskReconciler(subClient, "default-template", "ax-system")+ reconciler.SecretResolver = noSecrets+ reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond++ // Queue a reconcile and then a delete before the worker starts, as when a+ // task is deleted while the controller is still busy with other events.+ memStore := memory.NewStore()+ task := &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: "doomed", Atespace: "default"},+ Spec: &v1alpha1.TaskSpec{Image: "ghcr.io/test/img"},+ }+ if err := memStore.SaveTask(ctx, task); err != nil {+ t.Fatalf("failed to save task: %v", err)+ }+ if err := memStore.MarkTaskDeleting(ctx, "default", "doomed"); err != nil {+ t.Fatalf("MarkTaskDeleting failed: %v", err)+ }++ worker := controller.NewWorker(memStore, reconciler, "test-group", "worker-1")+ go func() { _ = worker.Run(ctx) }()++ deadline := time.Now().Add(3 * time.Second)+ for time.Now().Before(deadline) {+ if _, err := memStore.GetTask(ctx, "default", "doomed"); err != nil {+ break+ }+ time.Sleep(50 * time.Millisecond)+ }+ if _, err := memStore.GetTask(ctx, "default", "doomed"); err == nil {+ t.Fatalf("expected task record to be removed after cleanup")+ }+ if len(mockSrv.createdActors) != 0 || len(mockSrv.resumedActors) != 0 {+ t.Errorf("terminating task was reconciled: created %v, resumed %v", mockSrv.createdActors, mockSrv.resumedActors)+ }+}