Agent Executor (AX) is Google's open agentic orchestration runtime
Go
Remove health monitoring (#186)
Jaana Dogan2d02cf8a70bd
diff --git a/README.md b/README.mdindex 3474478..abf6c15 100644--- a/README.md+++ b/README.md@@ -11,7 +11,7 @@ AX, a short for Agent eXecutor, is a single-writer agent orchestrator system bui- **Local & Remote Agents**: Support for both in-process and remote agent deployment- **Streaming**: gRPC bidirectional streaming for agent communication- **Tools and Skills**: Built-in bash tool and agent skills support-- **Registry**: Agent discovery and automatic health monitoring+- **Registry**: Agent discoveryBuilt-in consistency and resumability features:- **Single-Writer Architecture**: Centralized controller ensures consistent state management@@ -268,9 +268,6 @@ eventlog:sqlite:filename: "eventlog/log.sqlite"-health_check:- enabled: true- interval: 30splanner:gemini:diff --git a/ax.yaml b/ax.yamlindex fa74ee5..069ebf6 100644--- a/ax.yaml+++ b/ax.yaml@@ -23,11 +23,6 @@ eventlog:sqlite:filename: "eventlog/log.sqlite"-health_check:- # Whether to enable active health checking- enabled: true- # Active health check interval for agents (e.g., "30s", "1m")- interval: 30s# Planner configuration (optional)planner:diff --git a/cmd/ax/serve.go b/cmd/ax/serve.goindex 3c6f2b7..d7d4195 100644--- a/cmd/ax/serve.go+++ b/cmd/ax/serve.go@@ -121,7 +121,6 @@ func newControllerFromConfig(ctx context.Context, cfg *config.Config) (*controllcontrollerConfig := controller.Config{EventLogBuilder: eventLogBuilder,PlannerBuilder: plannerBuilder,- HealthCheck: cfg.HealthCheck,}// Create controller@@ -131,7 +130,7 @@ func newControllerFromConfig(ctx context.Context, cfg *config.Config) (*controll}for _, agentCfg := range cfg.Registry.RemoteAgents {- if _, err := c.Registry().RegisterRemote(agentCfg); err != nil {+ if err := c.Registry().RegisterRemote(agentCfg); err != nil {return nil, fmt.Errorf("failed to register remote agent %s: %w", agentCfg.ID, err)}}diff --git a/cmd/ax/trace.go b/cmd/ax/trace.goindex e6c932b..1ee122b 100644--- a/cmd/ax/trace.go+++ b/cmd/ax/trace.go@@ -69,7 +69,7 @@ type TraceData struct {}var (- conversationID string+ conversationID stringtraceServerAddr stringtraceConfigFile string)@@ -124,8 +124,6 @@ func loadTraceData(ctx context.Context, convID string) (*TraceData, error) {return data, nil}--func fetchEventsByConversation(ctx context.Context, configPath string, convID string) ([]*proto.ExecutionEvent, string, error) {cfg, err := config.LoadFromFile(configPath)if err != nil {diff --git a/internal/config/config.go b/internal/config/config.goindex 90aed06..339250d 100644--- a/internal/config/config.go+++ b/internal/config/config.go@@ -18,7 +18,6 @@ package configimport ("fmt""os"- "time""github.com/google/ax/internal/agent""gopkg.in/yaml.v3"@@ -26,11 +25,10 @@ import (// Config represents the main configuration for AX server.type Config struct {- Server ServerConfig `yaml:"server"`- EventLog EventLogConfig `yaml:"eventlog"`- HealthCheck HealthCheckConfig `yaml:"health_check"`- Planner PlannerConfig `yaml:"planner,omitempty"`- Registry RegistryConfig `yaml:"registry,omitempty"`+ Server ServerConfig `yaml:"server"`+ EventLog EventLogConfig `yaml:"eventlog"`+ Planner PlannerConfig `yaml:"planner,omitempty"`+ Registry RegistryConfig `yaml:"registry,omitempty"`}// RegistryConfig allows registring agents.@@ -39,15 +37,6 @@ type RegistryConfig struct {KubernetesSandboxAgents []SandboxAgentConfig `yaml:"k8s_sandbox_agents,omitempty"`}-// HealthCheckConfig defines the configuration for agent health checks.-// When enabled, the controller will perform active polling to check the health of registered agents and-// flip the health status of agents that are not responsive. When disabled, the controller will not perform-// any health checks and will assume all agents are healthy.-type HealthCheckConfig struct {- Enabled bool `yaml:"enabled"` // default: false (no active polling)- Interval time.Duration `yaml:"interval"` // default: 30s-}-// ServerConfig configures the gRPC server.type ServerConfig struct {Address string `yaml:"address"` // Server address to listen on (e.g., ":8494")@@ -138,10 +127,6 @@ func (c *Config) setDefaults() {c.EventLog.SQLiteConfig.Filename = "eventlog/log.sqlite"}- // HealthCheck defaults- if c.HealthCheck.Enabled && c.HealthCheck.Interval == 0 {- c.HealthCheck.Interval = 30 * time.Second- }}// Validate validates the configuration.@@ -152,12 +137,6 @@ func (c *Config) Validate() error {if c.EventLog.SQLiteConfig.Filename == "" {return fmt.Errorf("eventlog.sqlite.filename is required")}- // Validate health check- if c.HealthCheck.Enabled {- if c.HealthCheck.Interval <= 0 {- return fmt.Errorf("health_check.interval must be positive when enabled")- }- }return nil}diff --git a/internal/controller/controller.go b/internal/controller/controller.goindex edd59bf..3982858 100644--- a/internal/controller/controller.go+++ b/internal/controller/controller.go@@ -24,7 +24,6 @@ import ("sync""github.com/google/ax/internal/agent"- "github.com/google/ax/internal/config""github.com/google/ax/internal/controller/executor""github.com/google/ax/internal/testagent""github.com/google/ax/proto"@@ -55,16 +54,12 @@ type PlannerBuilder func(ctx context.Context, r *Registry) (agent.Agent, error)type Config struct {EventLogBuilder executor.EventLogBuilderPlannerBuilder PlannerBuilder- HealthCheck config.HealthCheckConfig}// New creates a new controller instance.func New(ctx context.Context, config Config) (*Controller, error) {// Initialize agent registry- registry, err := NewRegistry(config.HealthCheck)- if err != nil {- return nil, fmt.Errorf("failed to create registry: %w", err)- }+ registry := NewRegistry()// Determine plan function// If no planner builder is provided, use the default Gemini planner.diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.goindex 002b412..7eee141 100644--- a/internal/controller/controller_test.go+++ b/internal/controller/controller_test.go@@ -164,7 +164,6 @@ func TestController_Exec_ResumptionAndIDGeneration(t *testing.T) {}}-func TestController_Exec_LastSeenSeq_Empty(t *testing.T) {ctx := context.Background()cid := "test-conv-seq"diff --git a/internal/controller/gemini_planner.go b/internal/controller/gemini_planner.goindex f0e0ce1..c59ad5a 100644--- a/internal/controller/gemini_planner.go+++ b/internal/controller/gemini_planner.go@@ -294,11 +294,11 @@ func (p *geminiPlannerAgent) handleConfirmationAnswer(inputs []*proto.Message) (// agentsToTools converts registry agents to Gemini function declarations.func agentsToTools(registry *Registry, nativeTools ...Tool) ([]*genai.Tool, error) {- healthyAgents := registry.ListHealthy()+ agents := registry.List()var tools []*genai.Tool// TODO(lhuan): Check if agentsToTools returns an error or empty list and return a friendly "no agent available, try later" error.- for _, id := range healthyAgents {+ for _, id := range agents {info, err := registry.GetInfo(id)if err != nil {continue // Skip agents we can't get info fordiff --git a/internal/controller/health.go b/internal/controller/health.godeleted file mode 100644index 1828b42..0000000--- a/internal/controller/health.go+++ /dev/null@@ -1,95 +0,0 @@-// Copyright 2026 Google LLC-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--package controller--import (- "sync"- "time"-- "github.com/google/ax/internal/config"-)--// HealthMonitor manages the periodic health checking of agents.-type HealthMonitor struct {- config config.HealthCheckConfig- registry *Registry- ticker *time.Ticker- stop chan struct{}- wg sync.WaitGroup- stopOnce sync.Once-}--// NewHealthMonitor creates a new health monitor.-func NewHealthMonitor(cfg config.HealthCheckConfig, r *Registry) *HealthMonitor {- return &HealthMonitor{- config: cfg,- registry: r,- stop: make(chan struct{}),- }-}--func (hm *HealthMonitor) Start() {- interval := hm.config.Interval- hm.ticker = time.NewTicker(interval)- hm.wg.Add(1)- go hm.run()-}--// Stop halts the health check loop.-func (hm *HealthMonitor) Stop() {- if hm.ticker != nil {- hm.ticker.Stop()- hm.stopOnce.Do(func() {- close(hm.stop)- })- hm.wg.Wait()- }-}--// run is the main loop for health checks.-func (hm *HealthMonitor) run() {- defer hm.wg.Done()-- for {- select {- case <-hm.stop:- return- case <-hm.ticker.C:- hm.performChecks()- }- }-}--// performChecks runs health checks for all agents.-func (hm *HealthMonitor) performChecks() {- // Get IDs safely- hm.registry.mu.RLock()- ids := make([]string, 0, len(hm.registry.agents))- for id := range hm.registry.agents {- ids = append(ids, id)- }- hm.registry.mu.RUnlock()-- // Run checks in parallel- var wg sync.WaitGroup- for _, id := range ids {- wg.Add(1)- go func(agentID string) {- defer wg.Done()- _ = hm.registry.healthCheck(agentID)- }(id)- }- wg.Wait()-}diff --git a/internal/controller/registry.go b/internal/controller/registry.goindex 5743d08..ae9596e 100644--- a/internal/controller/registry.go+++ b/internal/controller/registry.go@@ -17,9 +17,7 @@ package controllerimport ("context""fmt"- "log""sync"- "time""github.com/google/ax/internal/agent""github.com/google/ax/internal/config"@@ -36,13 +34,11 @@ const (// AgentInfo contains metadata about a registered agent.type AgentInfo struct {- ID string- Name string- Description string- Type AgentType- Healthy bool- LastHealthCheck time.Time- Metadata map[string]string+ ID string+ Name string+ Description string+ Type AgentType+ Metadata map[string]string}// Registry manages a collection of local and remote agents.@@ -51,25 +47,14 @@ type Registry struct {mu sync.RWMutexagents map[string]agent.AgentagentInfo map[string]*AgentInfo- monitor *HealthMonitor}// NewRegistry creates a new agent registry.-func NewRegistry(healthCheckConfig config.HealthCheckConfig) (*Registry, error) {- if healthCheckConfig.Enabled && healthCheckConfig.Interval <= 0 {- return nil, fmt.Errorf("invalid health check interval: %v", healthCheckConfig.Interval)- }-- r := &Registry{+func NewRegistry() *Registry {+ return &Registry{agents: make(map[string]agent.Agent),agentInfo: make(map[string]*AgentInfo),}- if healthCheckConfig.Enabled {- r.monitor = NewHealthMonitor(healthCheckConfig, r)- r.monitor.Start()- }-- return r, nil}func (r *Registry) Map() map[string]agent.Agent {@@ -94,29 +79,27 @@ func (r *Registry) RegisterLocal(cfg config.LocalAgentConfig) error {r.agents[cfg.ID] = cfg.Agentr.agentInfo[cfg.ID] = &AgentInfo{- ID: cfg.ID,- Name: cfg.Name,- Description: cfg.Description,- Type: AgentTypeLocal,- Healthy: true,- LastHealthCheck: time.Now(),- Metadata: cfg.Metadata,+ ID: cfg.ID,+ Name: cfg.Name,+ Description: cfg.Description,+ Type: AgentTypeLocal,+ Metadata: cfg.Metadata,}return nil}// RegisterRemote registers a remote agent by creating a remote agent client.-func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) (bool, error) {+func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) error {r.mu.Lock()defer r.mu.Unlock()if err := validateID(cfg.ID); err != nil {- return false, err+ return err}if _, ok := r.agents[cfg.ID]; ok {- return false, fmt.Errorf("agent %s already registered", cfg.ID)+ return fmt.Errorf("agent %s already registered", cfg.ID)}// Create remote agent client@@ -126,31 +109,19 @@ func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) (bool, error) {MaxRetries: 3,})if err != nil {- return false, fmt.Errorf("failed to create remote agent: %w", err)- }-- // Log a warning if the health check fails during registration, but don't fail the registration- // so that the server can still start up successfully.- healthy := true- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)- defer cancel()- if err := remoteAgent.HealthCheck(ctx); err != nil {- log.Printf("Warning: remote agent %s is unreachable or unhealthy during registration: %v", cfg.ID, err)- healthy = false+ return fmt.Errorf("failed to create remote agent: %w", err)}r.agents[cfg.ID] = remoteAgentr.agentInfo[cfg.ID] = &AgentInfo{- ID: cfg.ID,- Name: cfg.Name,- Description: cfg.Description,- Type: AgentTypeRemote,- Healthy: healthy,- LastHealthCheck: time.Now(),- Metadata: cfg.Metadata,+ ID: cfg.ID,+ Name: cfg.Name,+ Description: cfg.Description,+ Type: AgentTypeRemote,+ Metadata: cfg.Metadata,}- return healthy, nil+ return nil}// RegisterKubernetesSandbox registers a sandbox agent by dynamically provisioning a Sandbox on GKE.@@ -178,13 +149,11 @@ func (r *Registry) RegisterKubernetesSandbox(ctx context.Context, cfg config.Sanr.agents[cfg.ID] = sandboxAgentr.agentInfo[cfg.ID] = &AgentInfo{- ID: cfg.ID,- Name: cfg.Name,- Description: cfg.Description,- Type: AgentTypeKubernetesSandbox,- Healthy: true,- LastHealthCheck: time.Time{},- Metadata: cfg.Metadata,+ ID: cfg.ID,+ Name: cfg.Name,+ Description: cfg.Description,+ Type: AgentTypeKubernetesSandbox,+ Metadata: cfg.Metadata,}return nil@@ -229,52 +198,8 @@ func (r *Registry) List() []string {return ids}-// ListHealthy returns all healthy agent IDs.-func (r *Registry) ListHealthy() []string {- r.mu.RLock()- defer r.mu.RUnlock()-- ids := make([]string, 0)- for id, info := range r.agentInfo {- if info.Healthy {- ids = append(ids, id)- }- }-- return ids-}--// healthCheck performs a health check on a specific agent.-func (r *Registry) healthCheck(id string) error {- a, err := r.Get(id)- if err != nil {- return err- }-- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)- defer cancel()-- err = a.HealthCheck(ctx)-- r.mu.Lock()- defer r.mu.Unlock()-- info, exists := r.agentInfo[id]- if exists {- info.Healthy = (err == nil)- info.LastHealthCheck = time.Now()- }-- return err-}-// Close stops the registry and closes all agents.func (r *Registry) Close() error {- // Stop health checks before acquiring the lock to avoid deadlock- // (monitor.Stop waits for background routines that might need the lock)- if r.monitor != nil {- r.monitor.Stop()- }r.mu.Lock()defer r.mu.Unlock()diff --git a/internal/controller/registry_test.go b/internal/controller/registry_test.goindex e3e9ee1..59fd78a 100644--- a/internal/controller/registry_test.go+++ b/internal/controller/registry_test.go@@ -21,18 +21,13 @@ import ("fmt""net""testing"- "time""google.golang.org/grpc"- "github.com/google/ax/internal/agent""github.com/google/ax/internal/config"- "github.com/google/ax/internal/controller/executor""github.com/google/ax/proto")--type mockAgentServer struct {proto.UnimplementedAgentServiceServerhealthy bool@@ -60,138 +55,15 @@ func startMockGRPCServer(t *testing.T, healthy bool) (string, func()) {}}-func TestRegistry_HealthCheckScenarios(t *testing.T) {- tests := []struct {- name string- enabled bool- agentType string- expectHealthy bool- }{- {- name: "Local Agent (Always Healthy)",- enabled: true,- agentType: "local",- expectHealthy: true,- },- {- name: "Local Agent - Check Disabled (Always Healthy)",- enabled: false,- agentType: "local",- expectHealthy: true,- },- {- name: "Remote Agent - Check Enabled (Eventually Healthy)",- enabled: true,- agentType: "remote",- expectHealthy: true, // Eventually true- },- {- name: "Remote Agent - Check Disabled (Optimistically Healthy)",- enabled: false,- agentType: "remote",- expectHealthy: true, // Optimistically true- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- // Mock server for remote agent- address, cleanup := startMockGRPCServer(t, true)- defer cleanup()-- cfg := config.HealthCheckConfig{- Enabled: tt.enabled,- Interval: 10 * time.Millisecond,- }- r, err := NewRegistry(cfg)- if err != nil {- t.Fatalf("NewRegistry failed: %v", err)- }- defer r.Close()-- id := "test-agent"-- if tt.agentType == "local" {- // Mock local agent- agent := executor.AgentFunc(func(inputs []*proto.Message, tm agent.Executor, o agent.OutputHandler) {})- err = r.RegisterLocal(config.LocalAgentConfig{- ID: id,- Name: id,- Agent: agent,- })- if err != nil {- t.Fatalf("RegisterLocal failed: %v", err)- }- } else {- _, err = r.RegisterRemote(config.RemoteAgentConfig{- ID: id,- Name: id,- Address: address,- })- if err != nil {- t.Fatalf("RegisterRemote failed: %v", err)- }- }-- // Verify health- if tt.agentType == "remote" && tt.enabled {- // Must wait for health check- timeout := time.After(2 * time.Second)- ticker := time.NewTicker(20 * time.Millisecond)- defer ticker.Stop()-- success := false- for !success {- select {- case <-timeout:- t.Fatal("timed out waiting for agent to become healthy")- case <-ticker.C:- info, err := r.GetInfo(id)- if err != nil {- t.Fatalf("GetInfo failed: %v", err)- }- // Remote agents might start unhealthy until checked- if info.Healthy {- success = true- }- }- }- } else {- // Should be immediately healthy (local or disabled)- // Important: for disabled+remote, it is optimistically healthy.- // For local, it is always healthy (unless manually updated, but default is true).-- // Wait a tiny bit just in case async things happen, but usually sync for registration- time.Sleep(10 * time.Millisecond)-- info, err := r.GetInfo(id)- if err != nil {- t.Fatalf("GetInfo failed: %v", err)- }- if info.Healthy != tt.expectHealthy {- t.Errorf("expected healthy=%v, got %v", tt.expectHealthy, info.Healthy)- }- }- })- }-}-func TestRegistry_GracefulShutdown(t *testing.T) {- healthCheckConfig := config.HealthCheckConfig{- Enabled: true,- Interval: 10 * time.Millisecond,- }- r, regErr := NewRegistry(healthCheckConfig)- if regErr != nil {- t.Fatalf("NewRegistry failed: %v", regErr)- }+ r := NewRegistry()address, cleanup := startMockGRPCServer(t, true)defer cleanup()// Register multiple agents to create workloadfor i := range 50 {- _, err := r.RegisterRemote(config.RemoteAgentConfig{+ err := r.RegisterRemote(config.RemoteAgentConfig{ID: fmt.Sprintf("remote-shutdown-test-%d", i),Name: "Shutdown Test Remote",Address: address,@@ -201,9 +73,6 @@ func TestRegistry_GracefulShutdown(t *testing.T) {}}- // Let it run for a bit to ensure performChecks runs.- time.Sleep(50 * time.Millisecond)-// Close should return specific errors for failed agents, but NOT panic or deadlock// We are testing for absence of panic/deadlock here._ = r.Close()diff --git a/internal/server/server.go b/internal/server/server.goindex 2794ab7..d6c7319 100644--- a/internal/server/server.go+++ b/internal/server/server.go@@ -76,15 +76,13 @@ func (s *Server) RegisterAgent(ctx context.Context, req *proto.RegisterAgentRequregistry := s.controller.Registry()- var healthy boolswitch cfg := req.Config.(type) {case *proto.RegisterAgentRequest_Remote:if cfg.Remote.Address == "" {return nil, fmt.Errorf("address is required for remote agents")}- var err error- healthy, err = registry.RegisterRemote(config.RemoteAgentConfig{+ err := registry.RegisterRemote(config.RemoteAgentConfig{ID: req.AgentId,Name: req.Name,Description: req.Description,@@ -99,7 +97,7 @@ func (s *Server) RegisterAgent(ctx context.Context, req *proto.RegisterAgentRequ}return &proto.RegisterAgentResponse{- Healthy: healthy,+ Healthy: true,}, nil}diff --git a/internal/test_multi/test_multi.go b/internal/test_multi/test_multi.goindex ffca0d6..5c916bf 100644--- a/internal/test_multi/test_multi.go+++ b/internal/test_multi/test_multi.go@@ -43,9 +43,6 @@ func main() {// 2. Initialize controllerc, err := controller.New(ctx, controller.Config{- HealthCheck: config.HealthCheckConfig{- Enabled: false, // disable to speed up- },EventLogBuilder: func() (executor.EventLog, error) {return executor.OpenSQLiteEventLog(filepath.Join(os.TempDir(), "test_multi.db"))},@@ -69,7 +66,7 @@ func main() {}// 4. Register Remote Agent- if _, err := c.Registry().RegisterRemote(config.RemoteAgentConfig{+ if err := c.Registry().RegisterRemote(config.RemoteAgentConfig{ID: "remote-text-processor",Name: "Remote Text Processor",Description: "Adds the prefix 'Remote Prefix: ' to the text",