Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

Remove health monitoring (#186)

Jaana Dogan2d02cf8a70bd
diff --git a/README.md b/README.md
index 3474478..abf6c15 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ AX, a short for Agent eXecutor, is a single-writer agent orchestrator system bui
- **Local & Remote Agents**: Support for both in-process and remote agent deployment
- **Streaming**: gRPC bidirectional streaming for agent communication
- **Tools and Skills**: Built-in bash tool and agent skills support
-- **Registry**: Agent discovery and automatic health monitoring
+- **Registry**: Agent discovery
Built-in consistency and resumability features:
- **Single-Writer Architecture**: Centralized controller ensures consistent state management
@@ -268,9 +268,6 @@ eventlog:
sqlite:
filename: "eventlog/log.sqlite"
-health_check:
- enabled: true
- interval: 30s
planner:
gemini:
diff --git a/ax.yaml b/ax.yaml
index fa74ee5..069ebf6 100644
--- a/ax.yaml
+++ b/ax.yaml
@@ -23,11 +23,6 @@ eventlog:
sqlite:
filename: "eventlog/log.sqlite"
-health_check:
- # Whether to enable active health checking
- enabled: true
- # Active health check interval for agents (e.g., "30s", "1m")
- interval: 30s
# Planner configuration (optional)
planner:
diff --git a/cmd/ax/serve.go b/cmd/ax/serve.go
index 3c6f2b7..d7d4195 100644
--- a/cmd/ax/serve.go
+++ b/cmd/ax/serve.go
@@ -121,7 +121,6 @@ func newControllerFromConfig(ctx context.Context, cfg *config.Config) (*controll
controllerConfig := controller.Config{
EventLogBuilder: eventLogBuilder,
PlannerBuilder: plannerBuilder,
- HealthCheck: cfg.HealthCheck,
}
// Create controller
@@ -131,7 +130,7 @@ func newControllerFromConfig(ctx context.Context, cfg *config.Config) (*controll
}
for _, agentCfg := range cfg.Registry.RemoteAgents {
- if _, err := c.Registry().RegisterRemote(agentCfg); err != nil {
+ if err := c.Registry().RegisterRemote(agentCfg); err != nil {
return nil, fmt.Errorf("failed to register remote agent %s: %w", agentCfg.ID, err)
}
}
diff --git a/cmd/ax/trace.go b/cmd/ax/trace.go
index e6c932b..1ee122b 100644
--- a/cmd/ax/trace.go
+++ b/cmd/ax/trace.go
@@ -69,7 +69,7 @@ type TraceData struct {
}
var (
- conversationID string
+ conversationID string
traceServerAddr string
traceConfigFile string
)
@@ -124,8 +124,6 @@ func loadTraceData(ctx context.Context, convID string) (*TraceData, error) {
return data, nil
}
-
-
func fetchEventsByConversation(ctx context.Context, configPath string, convID string) ([]*proto.ExecutionEvent, string, error) {
cfg, err := config.LoadFromFile(configPath)
if err != nil {
diff --git a/internal/config/config.go b/internal/config/config.go
index 90aed06..339250d 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -18,7 +18,6 @@ package config
import (
"fmt"
"os"
- "time"
"github.com/google/ax/internal/agent"
"gopkg.in/yaml.v3"
@@ -26,11 +25,10 @@ import (
// Config represents the main configuration for AX server.
type Config struct {
- Server ServerConfig `yaml:"server"`
- EventLog EventLogConfig `yaml:"eventlog"`
- HealthCheck HealthCheckConfig `yaml:"health_check"`
- Planner PlannerConfig `yaml:"planner,omitempty"`
- Registry RegistryConfig `yaml:"registry,omitempty"`
+ Server ServerConfig `yaml:"server"`
+ EventLog EventLogConfig `yaml:"eventlog"`
+ Planner PlannerConfig `yaml:"planner,omitempty"`
+ Registry RegistryConfig `yaml:"registry,omitempty"`
}
// RegistryConfig allows registring agents.
@@ -39,15 +37,6 @@ type RegistryConfig struct {
KubernetesSandboxAgents []SandboxAgentConfig `yaml:"k8s_sandbox_agents,omitempty"`
}
-// HealthCheckConfig defines the configuration for agent health checks.
-// When enabled, the controller will perform active polling to check the health of registered agents and
-// flip the health status of agents that are not responsive. When disabled, the controller will not perform
-// any health checks and will assume all agents are healthy.
-type HealthCheckConfig struct {
- Enabled bool `yaml:"enabled"` // default: false (no active polling)
- Interval time.Duration `yaml:"interval"` // default: 30s
-}
-
// ServerConfig configures the gRPC server.
type ServerConfig struct {
Address string `yaml:"address"` // Server address to listen on (e.g., ":8494")
@@ -138,10 +127,6 @@ func (c *Config) setDefaults() {
c.EventLog.SQLiteConfig.Filename = "eventlog/log.sqlite"
}
- // HealthCheck defaults
- if c.HealthCheck.Enabled && c.HealthCheck.Interval == 0 {
- c.HealthCheck.Interval = 30 * time.Second
- }
}
// Validate validates the configuration.
@@ -152,12 +137,6 @@ func (c *Config) Validate() error {
if c.EventLog.SQLiteConfig.Filename == "" {
return fmt.Errorf("eventlog.sqlite.filename is required")
}
- // Validate health check
- if c.HealthCheck.Enabled {
- if c.HealthCheck.Interval <= 0 {
- return fmt.Errorf("health_check.interval must be positive when enabled")
- }
- }
return nil
}
diff --git a/internal/controller/controller.go b/internal/controller/controller.go
index edd59bf..3982858 100644
--- a/internal/controller/controller.go
+++ b/internal/controller/controller.go
@@ -24,7 +24,6 @@ import (
"sync"
"github.com/google/ax/internal/agent"
- "github.com/google/ax/internal/config"
"github.com/google/ax/internal/controller/executor"
"github.com/google/ax/internal/testagent"
"github.com/google/ax/proto"
@@ -55,16 +54,12 @@ type PlannerBuilder func(ctx context.Context, r *Registry) (agent.Agent, error)
type Config struct {
EventLogBuilder executor.EventLogBuilder
PlannerBuilder PlannerBuilder
- HealthCheck config.HealthCheckConfig
}
// New creates a new controller instance.
func New(ctx context.Context, config Config) (*Controller, error) {
// Initialize agent registry
- registry, err := NewRegistry(config.HealthCheck)
- if err != nil {
- return nil, fmt.Errorf("failed to create registry: %w", err)
- }
+ registry := NewRegistry()
// Determine plan function
// If no planner builder is provided, use the default Gemini planner.
diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go
index 002b412..7eee141 100644
--- a/internal/controller/controller_test.go
+++ b/internal/controller/controller_test.go
@@ -164,7 +164,6 @@ func TestController_Exec_ResumptionAndIDGeneration(t *testing.T) {
}
}
-
func TestController_Exec_LastSeenSeq_Empty(t *testing.T) {
ctx := context.Background()
cid := "test-conv-seq"
diff --git a/internal/controller/gemini_planner.go b/internal/controller/gemini_planner.go
index f0e0ce1..c59ad5a 100644
--- a/internal/controller/gemini_planner.go
+++ b/internal/controller/gemini_planner.go
@@ -294,11 +294,11 @@ func (p *geminiPlannerAgent) handleConfirmationAnswer(inputs []*proto.Message) (
// agentsToTools converts registry agents to Gemini function declarations.
func agentsToTools(registry *Registry, nativeTools ...Tool) ([]*genai.Tool, error) {
- healthyAgents := registry.ListHealthy()
+ agents := registry.List()
var tools []*genai.Tool
// TODO(lhuan): Check if agentsToTools returns an error or empty list and return a friendly "no agent available, try later" error.
- for _, id := range healthyAgents {
+ for _, id := range agents {
info, err := registry.GetInfo(id)
if err != nil {
continue // Skip agents we can't get info for
diff --git a/internal/controller/health.go b/internal/controller/health.go
deleted file mode 100644
index 1828b42..0000000
--- a/internal/controller/health.go
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package controller
-
-import (
- "sync"
- "time"
-
- "github.com/google/ax/internal/config"
-)
-
-// HealthMonitor manages the periodic health checking of agents.
-type HealthMonitor struct {
- config config.HealthCheckConfig
- registry *Registry
- ticker *time.Ticker
- stop chan struct{}
- wg sync.WaitGroup
- stopOnce sync.Once
-}
-
-// NewHealthMonitor creates a new health monitor.
-func NewHealthMonitor(cfg config.HealthCheckConfig, r *Registry) *HealthMonitor {
- return &HealthMonitor{
- config: cfg,
- registry: r,
- stop: make(chan struct{}),
- }
-}
-
-func (hm *HealthMonitor) Start() {
- interval := hm.config.Interval
- hm.ticker = time.NewTicker(interval)
- hm.wg.Add(1)
- go hm.run()
-}
-
-// Stop halts the health check loop.
-func (hm *HealthMonitor) Stop() {
- if hm.ticker != nil {
- hm.ticker.Stop()
- hm.stopOnce.Do(func() {
- close(hm.stop)
- })
- hm.wg.Wait()
- }
-}
-
-// run is the main loop for health checks.
-func (hm *HealthMonitor) run() {
- defer hm.wg.Done()
-
- for {
- select {
- case <-hm.stop:
- return
- case <-hm.ticker.C:
- hm.performChecks()
- }
- }
-}
-
-// performChecks runs health checks for all agents.
-func (hm *HealthMonitor) performChecks() {
- // Get IDs safely
- hm.registry.mu.RLock()
- ids := make([]string, 0, len(hm.registry.agents))
- for id := range hm.registry.agents {
- ids = append(ids, id)
- }
- hm.registry.mu.RUnlock()
-
- // Run checks in parallel
- var wg sync.WaitGroup
- for _, id := range ids {
- wg.Add(1)
- go func(agentID string) {
- defer wg.Done()
- _ = hm.registry.healthCheck(agentID)
- }(id)
- }
- wg.Wait()
-}
diff --git a/internal/controller/registry.go b/internal/controller/registry.go
index 5743d08..ae9596e 100644
--- a/internal/controller/registry.go
+++ b/internal/controller/registry.go
@@ -17,9 +17,7 @@ package controller
import (
"context"
"fmt"
- "log"
"sync"
- "time"
"github.com/google/ax/internal/agent"
"github.com/google/ax/internal/config"
@@ -36,13 +34,11 @@ const (
// AgentInfo contains metadata about a registered agent.
type AgentInfo struct {
- ID string
- Name string
- Description string
- Type AgentType
- Healthy bool
- LastHealthCheck time.Time
- Metadata map[string]string
+ ID string
+ Name string
+ Description string
+ Type AgentType
+ Metadata map[string]string
}
// Registry manages a collection of local and remote agents.
@@ -51,25 +47,14 @@ type Registry struct {
mu sync.RWMutex
agents map[string]agent.Agent
agentInfo map[string]*AgentInfo
- monitor *HealthMonitor
}
// NewRegistry creates a new agent registry.
-func NewRegistry(healthCheckConfig config.HealthCheckConfig) (*Registry, error) {
- if healthCheckConfig.Enabled && healthCheckConfig.Interval <= 0 {
- return nil, fmt.Errorf("invalid health check interval: %v", healthCheckConfig.Interval)
- }
-
- r := &Registry{
+func NewRegistry() *Registry {
+ return &Registry{
agents: make(map[string]agent.Agent),
agentInfo: make(map[string]*AgentInfo),
}
- if healthCheckConfig.Enabled {
- r.monitor = NewHealthMonitor(healthCheckConfig, r)
- r.monitor.Start()
- }
-
- return r, nil
}
func (r *Registry) Map() map[string]agent.Agent {
@@ -94,29 +79,27 @@ func (r *Registry) RegisterLocal(cfg config.LocalAgentConfig) error {
r.agents[cfg.ID] = cfg.Agent
r.agentInfo[cfg.ID] = &AgentInfo{
- ID: cfg.ID,
- Name: cfg.Name,
- Description: cfg.Description,
- Type: AgentTypeLocal,
- Healthy: true,
- LastHealthCheck: time.Now(),
- Metadata: cfg.Metadata,
+ ID: cfg.ID,
+ Name: cfg.Name,
+ Description: cfg.Description,
+ Type: AgentTypeLocal,
+ Metadata: cfg.Metadata,
}
return nil
}
// RegisterRemote registers a remote agent by creating a remote agent client.
-func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) (bool, error) {
+func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) error {
r.mu.Lock()
defer r.mu.Unlock()
if err := validateID(cfg.ID); err != nil {
- return false, err
+ return err
}
if _, ok := r.agents[cfg.ID]; ok {
- return false, fmt.Errorf("agent %s already registered", cfg.ID)
+ return fmt.Errorf("agent %s already registered", cfg.ID)
}
// Create remote agent client
@@ -126,31 +109,19 @@ func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) (bool, error) {
MaxRetries: 3,
})
if err != nil {
- return false, fmt.Errorf("failed to create remote agent: %w", err)
- }
-
- // Log a warning if the health check fails during registration, but don't fail the registration
- // so that the server can still start up successfully.
- healthy := true
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- if err := remoteAgent.HealthCheck(ctx); err != nil {
- log.Printf("Warning: remote agent %s is unreachable or unhealthy during registration: %v", cfg.ID, err)
- healthy = false
+ return fmt.Errorf("failed to create remote agent: %w", err)
}
r.agents[cfg.ID] = remoteAgent
r.agentInfo[cfg.ID] = &AgentInfo{
- ID: cfg.ID,
- Name: cfg.Name,
- Description: cfg.Description,
- Type: AgentTypeRemote,
- Healthy: healthy,
- LastHealthCheck: time.Now(),
- Metadata: cfg.Metadata,
+ ID: cfg.ID,
+ Name: cfg.Name,
+ Description: cfg.Description,
+ Type: AgentTypeRemote,
+ Metadata: cfg.Metadata,
}
- return healthy, nil
+ return nil
}
// RegisterKubernetesSandbox registers a sandbox agent by dynamically provisioning a Sandbox on GKE.
@@ -178,13 +149,11 @@ func (r *Registry) RegisterKubernetesSandbox(ctx context.Context, cfg config.San
r.agents[cfg.ID] = sandboxAgent
r.agentInfo[cfg.ID] = &AgentInfo{
- ID: cfg.ID,
- Name: cfg.Name,
- Description: cfg.Description,
- Type: AgentTypeKubernetesSandbox,
- Healthy: true,
- LastHealthCheck: time.Time{},
- Metadata: cfg.Metadata,
+ ID: cfg.ID,
+ Name: cfg.Name,
+ Description: cfg.Description,
+ Type: AgentTypeKubernetesSandbox,
+ Metadata: cfg.Metadata,
}
return nil
@@ -229,52 +198,8 @@ func (r *Registry) List() []string {
return ids
}
-// ListHealthy returns all healthy agent IDs.
-func (r *Registry) ListHealthy() []string {
- r.mu.RLock()
- defer r.mu.RUnlock()
-
- ids := make([]string, 0)
- for id, info := range r.agentInfo {
- if info.Healthy {
- ids = append(ids, id)
- }
- }
-
- return ids
-}
-
-// healthCheck performs a health check on a specific agent.
-func (r *Registry) healthCheck(id string) error {
- a, err := r.Get(id)
- if err != nil {
- return err
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- err = a.HealthCheck(ctx)
-
- r.mu.Lock()
- defer r.mu.Unlock()
-
- info, exists := r.agentInfo[id]
- if exists {
- info.Healthy = (err == nil)
- info.LastHealthCheck = time.Now()
- }
-
- return err
-}
-
// Close stops the registry and closes all agents.
func (r *Registry) Close() error {
- // Stop health checks before acquiring the lock to avoid deadlock
- // (monitor.Stop waits for background routines that might need the lock)
- if r.monitor != nil {
- r.monitor.Stop()
- }
r.mu.Lock()
defer r.mu.Unlock()
diff --git a/internal/controller/registry_test.go b/internal/controller/registry_test.go
index e3e9ee1..59fd78a 100644
--- a/internal/controller/registry_test.go
+++ b/internal/controller/registry_test.go
@@ -21,18 +21,13 @@ import (
"fmt"
"net"
"testing"
- "time"
"google.golang.org/grpc"
- "github.com/google/ax/internal/agent"
"github.com/google/ax/internal/config"
- "github.com/google/ax/internal/controller/executor"
"github.com/google/ax/proto"
)
-
-
type mockAgentServer struct {
proto.UnimplementedAgentServiceServer
healthy bool
@@ -60,138 +55,15 @@ func startMockGRPCServer(t *testing.T, healthy bool) (string, func()) {
}
}
-func TestRegistry_HealthCheckScenarios(t *testing.T) {
- tests := []struct {
- name string
- enabled bool
- agentType string
- expectHealthy bool
- }{
- {
- name: "Local Agent (Always Healthy)",
- enabled: true,
- agentType: "local",
- expectHealthy: true,
- },
- {
- name: "Local Agent - Check Disabled (Always Healthy)",
- enabled: false,
- agentType: "local",
- expectHealthy: true,
- },
- {
- name: "Remote Agent - Check Enabled (Eventually Healthy)",
- enabled: true,
- agentType: "remote",
- expectHealthy: true, // Eventually true
- },
- {
- name: "Remote Agent - Check Disabled (Optimistically Healthy)",
- enabled: false,
- agentType: "remote",
- expectHealthy: true, // Optimistically true
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- // Mock server for remote agent
- address, cleanup := startMockGRPCServer(t, true)
- defer cleanup()
-
- cfg := config.HealthCheckConfig{
- Enabled: tt.enabled,
- Interval: 10 * time.Millisecond,
- }
- r, err := NewRegistry(cfg)
- if err != nil {
- t.Fatalf("NewRegistry failed: %v", err)
- }
- defer r.Close()
-
- id := "test-agent"
-
- if tt.agentType == "local" {
- // Mock local agent
- agent := executor.AgentFunc(func(inputs []*proto.Message, tm agent.Executor, o agent.OutputHandler) {})
- err = r.RegisterLocal(config.LocalAgentConfig{
- ID: id,
- Name: id,
- Agent: agent,
- })
- if err != nil {
- t.Fatalf("RegisterLocal failed: %v", err)
- }
- } else {
- _, err = r.RegisterRemote(config.RemoteAgentConfig{
- ID: id,
- Name: id,
- Address: address,
- })
- if err != nil {
- t.Fatalf("RegisterRemote failed: %v", err)
- }
- }
-
- // Verify health
- if tt.agentType == "remote" && tt.enabled {
- // Must wait for health check
- timeout := time.After(2 * time.Second)
- ticker := time.NewTicker(20 * time.Millisecond)
- defer ticker.Stop()
-
- success := false
- for !success {
- select {
- case <-timeout:
- t.Fatal("timed out waiting for agent to become healthy")
- case <-ticker.C:
- info, err := r.GetInfo(id)
- if err != nil {
- t.Fatalf("GetInfo failed: %v", err)
- }
- // Remote agents might start unhealthy until checked
- if info.Healthy {
- success = true
- }
- }
- }
- } else {
- // Should be immediately healthy (local or disabled)
- // Important: for disabled+remote, it is optimistically healthy.
- // For local, it is always healthy (unless manually updated, but default is true).
-
- // Wait a tiny bit just in case async things happen, but usually sync for registration
- time.Sleep(10 * time.Millisecond)
-
- info, err := r.GetInfo(id)
- if err != nil {
- t.Fatalf("GetInfo failed: %v", err)
- }
- if info.Healthy != tt.expectHealthy {
- t.Errorf("expected healthy=%v, got %v", tt.expectHealthy, info.Healthy)
- }
- }
- })
- }
-}
-
func TestRegistry_GracefulShutdown(t *testing.T) {
- healthCheckConfig := config.HealthCheckConfig{
- Enabled: true,
- Interval: 10 * time.Millisecond,
- }
- r, regErr := NewRegistry(healthCheckConfig)
- if regErr != nil {
- t.Fatalf("NewRegistry failed: %v", regErr)
- }
+ r := NewRegistry()
address, cleanup := startMockGRPCServer(t, true)
defer cleanup()
// Register multiple agents to create workload
for i := range 50 {
- _, err := r.RegisterRemote(config.RemoteAgentConfig{
+ err := r.RegisterRemote(config.RemoteAgentConfig{
ID: fmt.Sprintf("remote-shutdown-test-%d", i),
Name: "Shutdown Test Remote",
Address: address,
@@ -201,9 +73,6 @@ func TestRegistry_GracefulShutdown(t *testing.T) {
}
}
- // Let it run for a bit to ensure performChecks runs.
- time.Sleep(50 * time.Millisecond)
-
// Close should return specific errors for failed agents, but NOT panic or deadlock
// We are testing for absence of panic/deadlock here.
_ = r.Close()
diff --git a/internal/server/server.go b/internal/server/server.go
index 2794ab7..d6c7319 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -76,15 +76,13 @@ func (s *Server) RegisterAgent(ctx context.Context, req *proto.RegisterAgentRequ
registry := s.controller.Registry()
- var healthy bool
switch cfg := req.Config.(type) {
case *proto.RegisterAgentRequest_Remote:
if cfg.Remote.Address == "" {
return nil, fmt.Errorf("address is required for remote agents")
}
- var err error
- healthy, err = registry.RegisterRemote(config.RemoteAgentConfig{
+ err := registry.RegisterRemote(config.RemoteAgentConfig{
ID: req.AgentId,
Name: req.Name,
Description: req.Description,
@@ -99,7 +97,7 @@ func (s *Server) RegisterAgent(ctx context.Context, req *proto.RegisterAgentRequ
}
return &proto.RegisterAgentResponse{
- Healthy: healthy,
+ Healthy: true,
}, nil
}
diff --git a/internal/test_multi/test_multi.go b/internal/test_multi/test_multi.go
index ffca0d6..5c916bf 100644
--- a/internal/test_multi/test_multi.go
+++ b/internal/test_multi/test_multi.go
@@ -43,9 +43,6 @@ func main() {
// 2. Initialize controller
c, err := controller.New(ctx, controller.Config{
- HealthCheck: config.HealthCheckConfig{
- Enabled: false, // disable to speed up
- },
EventLogBuilder: func() (executor.EventLog, error) {
return executor.OpenSQLiteEventLog(filepath.Join(os.TempDir(), "test_multi.db"))
},
@@ -69,7 +66,7 @@ func main() {
}
// 4. Register Remote Agent
- if _, err := c.Registry().RegisterRemote(config.RemoteAgentConfig{
+ if err := c.Registry().RegisterRemote(config.RemoteAgentConfig{
ID: "remote-text-processor",
Name: "Remote Text Processor",
Description: "Adds the prefix 'Remote Prefix: ' to the text",