Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

allow disable active health check + encapsulate health monitor into a separate class (#15)

JoyceLiu7a855eb7654a
diff --git a/cmd/gar/serve.go b/cmd/gar/serve.go
index 451aeeb..7541b73 100644
--- a/cmd/gar/serve.go
+++ b/cmd/gar/serve.go
@@ -112,10 +112,10 @@ func newControllerFromConfig(ctx context.Context, cfg *config.Config) (*controll
// Build controller config
controllerConfig := controller.Config{
- EventLogFactory: eventLogFactory,
- PlannerFactory: plannerFactory,
- MaxSteps: cfg.MaxSteps,
- HealthCheckInterval: cfg.HealthCheck.Interval,
+ EventLogFactory: eventLogFactory,
+ PlannerFactory: plannerFactory,
+ MaxSteps: cfg.MaxSteps,
+ HealthCheck: cfg.HealthCheck,
}
// Create controller
diff --git a/examples/local_agent/main.go b/examples/local_agent/main.go
index b3039c1..3bdd38d 100644
--- a/examples/local_agent/main.go
+++ b/examples/local_agent/main.go
@@ -47,8 +47,11 @@ func main() {
}
c, err := controller.New(ctx, controller.Config{
- MaxSteps: 10,
- HealthCheckInterval: 30 * time.Second,
+ MaxSteps: 10,
+ HealthCheck: config.HealthCheckConfig{
+ Enabled: true,
+ Interval: 30 * time.Second,
+ },
})
if err != nil {
log.Fatalf("Error creating controller: %v\n", err)
diff --git a/gar.yaml b/gar.yaml
index 5b4277e..eac0d4a 100644
--- a/gar.yaml
+++ b/gar.yaml
@@ -26,7 +26,9 @@ eventlog:
max_steps: 50
health_check:
- # Health check interval for agents (e.g., "30s", "1m")
+ # Whether to enable active health checking
+ enabled: true
+ # Active health check interval for agents (e.g., "30s", "1m")
interval: 30s
# Planner configuration (optional)
diff --git a/internal/config/config.go b/internal/config/config.go
index b51b175..d9c618b 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -34,9 +34,14 @@ type Config struct {
RemoteAgents []RemoteAgentConfig `yaml:"remote_agents,omitempty"` // List of remote agents to register
}
-// PlannerConfig configures the planner.
-type PlannerConfig struct {
- Gemini GeminiPlannerConfig `yaml:"gemini"`
+// HealthCheckConfig defines the configuration for agent health checks.
+// When enabled, the controller will perform active polling to check the health of registered agents and
+// flip the health status of agents that are not responsive. When disabled, the controller will not perform
+// any health checks and will assume all agents are healthy.
+// TODO(lhuan): Add passive health checks and discovery rules.
+type HealthCheckConfig struct {
+ Enabled bool `yaml:"enabled"` // default: false (no active polling)
+ Interval time.Duration `yaml:"interval"` // default: 30s
}
// ServerConfig configures the gRPC server.
@@ -49,8 +54,9 @@ type EventLogConfig struct {
Dir string `yaml:"dir"` // Directory for event log files
}
-type HealthCheckConfig struct {
- Interval time.Duration `yaml:"interval"` // Interval between health checks
+// PlannerConfig configures the planner.
+type PlannerConfig struct {
+ Gemini GeminiPlannerConfig `yaml:"gemini"`
}
// GeminiPlannerConfig configures the Gemini-based planner.
@@ -116,8 +122,8 @@ func (c *Config) setDefaults() {
if c.MaxSteps == 0 {
c.MaxSteps = 100
}
-
- if c.HealthCheck.Interval == 0 {
+ // HealthCheck defaults
+ if c.HealthCheck.Enabled && c.HealthCheck.Interval == 0 {
c.HealthCheck.Interval = 30 * time.Second
}
}
@@ -133,8 +139,13 @@ func (c *Config) Validate() error {
if c.MaxSteps <= 0 {
return fmt.Errorf("max_steps must be positive")
}
- if c.HealthCheck.Interval <= 0 {
- return fmt.Errorf("health_check.interval must be positive")
+
+ // Validate health check
+ if c.HealthCheck.Enabled {
+ if c.HealthCheck.Interval <= 0 {
+ return fmt.Errorf("health_check.interval must be positive when enabled")
+ }
}
+
return nil
}
diff --git a/internal/controller/controller.go b/internal/controller/controller.go
index 5dc894e..81ab44b 100644
--- a/internal/controller/controller.go
+++ b/internal/controller/controller.go
@@ -20,9 +20,9 @@ import (
"context"
"fmt"
"sync"
- "time"
"github.com/google/gar/agent"
+ "github.com/google/gar/internal/config"
"github.com/google/gar/internal/eventlog"
"github.com/google/gar/proto"
)
@@ -32,7 +32,6 @@ import (
type Controller struct {
inFlightSessionsMu sync.Mutex
inFlightSessions map[string]struct{}
-
sessionManager *SessionManager
registry *Registry
loopExecutor *LoopExecutor
@@ -46,8 +45,8 @@ type Config struct {
EventLogFactory eventlog.EventLogFactory
PlannerFactory PlannerFactory
// TODO(jbd): Add CompactionFunc.
- HealthCheckInterval time.Duration
- MaxSteps int
+ HealthCheck config.HealthCheckConfig
+ MaxSteps int
}
// New creates a new controller instance.
@@ -55,9 +54,7 @@ func New(ctx context.Context, config Config) (*Controller, error) {
if config.MaxSteps == 0 {
config.MaxSteps = 100
}
- if config.HealthCheckInterval == 0 {
- config.HealthCheckInterval = 30 * time.Second
- }
+
if config.EventLogFactory == nil {
config.EventLogFactory = func(sessionID string) (eventlog.EventLog, error) {
return eventlog.NewFileEventLog(eventlog.FileConfig{
@@ -71,7 +68,10 @@ func New(ctx context.Context, config Config) (*Controller, error) {
sessionManager := NewSessionManager(config.EventLogFactory)
// Initialize agent registry
- registry := NewRegistry(config.HealthCheckInterval)
+ registry, err := NewRegistry(config.HealthCheck)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create registry: %w", err)
+ }
// Determine plan function
// If no planner factory is provided, use the default Gemini planner.
diff --git a/internal/controller/gemini.go b/internal/controller/gemini.go
index f7555ef..2170d78 100644
--- a/internal/controller/gemini.go
+++ b/internal/controller/gemini.go
@@ -140,6 +140,7 @@ func agentsToTools(registry *Registry) ([]*genai.Tool, error) {
}
var tools []*genai.Tool
+ // TODO(lhuan): Check if agentsToTools returns an error or empty list and return a friendly "no agent available, try later" error.
for _, id := range healthyAgents {
info, err := registry.GetInfo(id)
if err != nil {
diff --git a/internal/controller/health.go b/internal/controller/health.go
new file mode 100644
index 0000000..80e92ad
--- /dev/null
+++ b/internal/controller/health.go
@@ -0,0 +1,78 @@
+package controller
+
+import (
+ "sync"
+ "time"
+
+ "github.com/google/gar/internal/config"
+)
+
+// HealthMonitor manages the periodic health checking of agents.
+type HealthMonitor struct {
+ config config.HealthCheckConfig
+ registry *Registry
+ ticker *time.Ticker
+ stop chan struct{}
+ wg sync.WaitGroup
+}
+
+// NewHealthMonitor creates a new health monitor.
+func NewHealthMonitor(cfg config.HealthCheckConfig, r *Registry) *HealthMonitor {
+ return &HealthMonitor{
+ config: cfg,
+ registry: r,
+ stop: make(chan struct{}),
+ }
+}
+
+func (hm *HealthMonitor) Start() {
+ interval := hm.config.Interval
+ hm.ticker = time.NewTicker(interval)
+ hm.wg.Add(1)
+ go hm.run()
+}
+
+// Stop halts the health check loop.
+func (hm *HealthMonitor) Stop() {
+ if hm.ticker != nil {
+ hm.ticker.Stop()
+ close(hm.stop)
+ hm.wg.Wait()
+ }
+}
+
+// run is the main loop for health checks.
+func (hm *HealthMonitor) run() {
+ defer hm.wg.Done()
+
+ for {
+ select {
+ case <-hm.stop:
+ return
+ case <-hm.ticker.C:
+ hm.performChecks()
+ }
+ }
+}
+
+// performChecks triggers health checks for all agents.
+func (hm *HealthMonitor) performChecks() {
+ // Get IDs safely
+ hm.registry.mu.RLock()
+ ids := make([]string, 0, len(hm.registry.agents))
+ for id := range hm.registry.agents {
+ ids = append(ids, id)
+ }
+ hm.registry.mu.RUnlock()
+
+ // Run checks in parallel
+ var wg sync.WaitGroup
+ for _, id := range ids {
+ wg.Add(1)
+ go func(agentID string) {
+ defer wg.Done()
+ _ = hm.registry.healthCheck(agentID)
+ }(id)
+ }
+ wg.Wait()
+}
diff --git a/internal/controller/loop.go b/internal/controller/loop.go
index fb4c90c..9fd174f 100644
--- a/internal/controller/loop.go
+++ b/internal/controller/loop.go
@@ -145,6 +145,8 @@ func (e *LoopExecutor) runTask(ctx context.Context, session *Session, task *Task
return fmt.Errorf("failed to get agent: %w", err)
}
+ // TODO(lhuan): Handle scenario where agent is marked healthy (optimistic) but fails to respond (e.g. still starting up).
+ // Return "agent internal error, try again later" to the user.
if err := ag.Process(ctx, session.ID(), &proto.ProcessRequest{
Contents: task.Inputs,
}, taskOutputHandler); err != nil {
diff --git a/internal/controller/registry.go b/internal/controller/registry.go
index 8122e14..3ac5102 100644
--- a/internal/controller/registry.go
+++ b/internal/controller/registry.go
@@ -46,32 +46,28 @@ type AgentInfo struct {
// Registry manages a collection of local and remote agents.
// It provides agent discovery, health monitoring, and load balancing.
type Registry struct {
- mu sync.RWMutex
- agents map[string]agent.Agent
- agentInfo map[string]*AgentInfo
- healthCheckTicker *time.Ticker
- stopHealthCheck chan struct{}
+ mu sync.RWMutex
+ agents map[string]agent.Agent
+ agentInfo map[string]*AgentInfo
+ monitor *HealthMonitor
}
// NewRegistry creates a new agent registry.
-func NewRegistry(healthCheckInterval time.Duration) *Registry {
- if healthCheckInterval == 0 {
- healthCheckInterval = 30 * time.Second
+func NewRegistry(healthCheckConfig config.HealthCheckConfig) (*Registry, error) {
+ if healthCheckConfig.Enabled && healthCheckConfig.Interval <= 0 {
+ return nil, fmt.Errorf("invalid health check interval: %v", healthCheckConfig.Interval)
}
r := &Registry{
- agents: make(map[string]agent.Agent),
- agentInfo: make(map[string]*AgentInfo),
- stopHealthCheck: make(chan struct{}),
+ agents: make(map[string]agent.Agent),
+ agentInfo: make(map[string]*AgentInfo),
}
-
- // Start background health check if interval is positive
- if healthCheckInterval > 0 {
- r.healthCheckTicker = time.NewTicker(healthCheckInterval)
- go r.runHealthChecks()
+ if healthCheckConfig.Enabled {
+ r.monitor = NewHealthMonitor(healthCheckConfig, r)
+ r.monitor.Start()
}
- return r
+ return r, nil
}
// RegisterLocal registers a local (in-process) agent.
@@ -102,6 +98,8 @@ func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) error {
r.mu.Lock()
defer r.mu.Unlock()
+ // TODO(lhuan): Consider enforcing health check during registration. Only allow registration if the agent is reachable and healthy.
+
if _, ok := r.agents[cfg.ID]; ok {
return fmt.Errorf("agent %s already registered", cfg.ID)
}
@@ -122,7 +120,7 @@ func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) error {
Name: cfg.Name,
Description: cfg.Description,
Type: AgentTypeRemote,
- Healthy: false, // Will be checked by health monitor
+ Healthy: (r.monitor == nil), // Optimistic if no health check, pessimistic if health check is enabled
LastHealthCheck: time.Time{},
Metadata: cfg.Metadata,
}
@@ -169,8 +167,8 @@ func (r *Registry) GetInfo(id string) (*AgentInfo, error) {
r.mu.RLock()
defer r.mu.RUnlock()
- info, exists := r.agentInfo[id]
- if !exists {
+ info, ok := r.agentInfo[id]
+ if !ok {
return nil, fmt.Errorf("agent %s not found", id)
}
@@ -205,6 +203,7 @@ func (r *Registry) ListHealthy() []string {
return ids
}
+
// healthCheck performs a health check on a specific agent.
func (r *Registry) healthCheck(id string) error {
a, err := r.Get(id)
@@ -229,38 +228,18 @@ func (r *Registry) healthCheck(id string) error {
return err
}
-// runHealthChecks runs periodic health checks on all agents.
-func (r *Registry) runHealthChecks() {
- for {
- select {
- case <-r.healthCheckTicker.C:
- r.performHealthChecks()
- case <-r.stopHealthCheck:
- return
- }
- }
-}
-
-// performHealthChecks checks the health of all registered agents.
-func (r *Registry) performHealthChecks() {
- ids := r.List()
- for _, id := range ids {
- // Run health check (ignore errors, status is updated in HealthCheck method)
- _ = r.healthCheck(id)
- }
-}
// Close stops the registry and closes all agents.
func (r *Registry) Close() error {
+ // Stop health checks before acquiring the lock to avoid deadlock
+ // (monitor.Stop waits for background routines that might need the lock)
+ if r.monitor != nil {
+ r.monitor.Stop()
+ }
+
r.mu.Lock()
defer r.mu.Unlock()
- // Stop health checks
- if r.healthCheckTicker != nil {
- r.healthCheckTicker.Stop()
- close(r.stopHealthCheck)
- }
-
// Close all agents
var firstErr error
for id, a := range r.agents {
diff --git a/internal/controller/registry_test.go b/internal/controller/registry_test.go
new file mode 100644
index 0000000..d9287c8
--- /dev/null
+++ b/internal/controller/registry_test.go
@@ -0,0 +1,232 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controller
+
+// TODO(lhuan): Setup a better automated testing framework
+
+import (
+ "context"
+ "net"
+ "testing"
+ "time"
+
+ "google.golang.org/grpc"
+
+ "github.com/google/gar/agent"
+ "github.com/google/gar/internal/config"
+ "github.com/google/gar/proto"
+)
+
+// MockAgent is a mock implementation of the Agent interface for testing.
+type MockAgent struct {
+ processFunc func(ctx context.Context, sessionID string, req *proto.ProcessRequest, handler agent.OutputHandler) error
+ healthCheckFunc func(ctx context.Context) error
+ closeFunc func() error
+}
+
+func (m *MockAgent) Process(ctx context.Context, sessionID string, req *proto.ProcessRequest, handler agent.OutputHandler) error {
+ if m.processFunc != nil {
+ return m.processFunc(ctx, sessionID, req, handler)
+ }
+ return nil
+}
+
+func (m *MockAgent) HealthCheck(ctx context.Context) error {
+ if m.healthCheckFunc != nil {
+ return m.healthCheckFunc(ctx)
+ }
+ return nil
+}
+
+func (m *MockAgent) Close() error {
+ if m.closeFunc != nil {
+ return m.closeFunc()
+ }
+ return nil
+}
+
+type mockAgentServer struct {
+ proto.UnimplementedAgentServiceServer
+ healthy bool
+}
+
+func (s *mockAgentServer) HealthCheck(ctx context.Context, req *proto.HealthCheckRequest) (*proto.HealthCheckResponse, error) {
+ return &proto.HealthCheckResponse{Healthy: s.healthy}, nil
+}
+
+func startMockGRPCServer(t *testing.T, healthy bool) (string, func()) {
+ lis, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ t.Fatalf("failed to listen: %v", err)
+ }
+ s := grpc.NewServer()
+ proto.RegisterAgentServiceServer(s, &mockAgentServer{healthy: healthy})
+ go func() {
+ if err := s.Serve(lis); err != nil {
+ // server might be closed
+ }
+ }()
+ return lis.Addr().String(), func() {
+ s.Stop()
+ lis.Close()
+ }
+}
+
+
+
+func TestRegistry_HealthCheckScenarios(t *testing.T) {
+ tests := []struct {
+ name string
+ enabled bool
+ agentType string
+ expectHealthy bool
+ }{
+ {
+ name: "Local Agent (Always Healthy)",
+ enabled: true,
+ agentType: "local",
+ expectHealthy: true,
+ },
+ {
+ name: "Local Agent - Check Disabled (Always Healthy)",
+ enabled: false,
+ agentType: "local",
+ expectHealthy: true,
+ },
+ {
+ name: "Remote Agent - Check Enabled (Eventually Healthy)",
+ enabled: true,
+ agentType: "remote",
+ expectHealthy: true, // Eventually true
+ },
+ {
+ name: "Remote Agent - Check Disabled (Optimistically Healthy)",
+ enabled: false,
+ agentType: "remote",
+ expectHealthy: true, // Optimistically true
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Mock server for remote agent
+ address, cleanup := startMockGRPCServer(t, true)
+ defer cleanup()
+
+ cfg := config.HealthCheckConfig{
+ Enabled: tt.enabled,
+ Interval: 10 * time.Millisecond,
+ }
+ r, err := NewRegistry(cfg)
+ if err != nil {
+ t.Fatalf("NewRegistry failed: %v", err)
+ }
+ defer r.Close()
+
+ id := "test-agent"
+
+ if tt.agentType == "local" {
+ // Mock local agent
+ validMockAgent := &MockAgent{}
+ err = r.RegisterLocal(config.LocalAgentConfig{
+ ID: id,
+ Name: id,
+ Agent: validMockAgent,
+ })
+ if err != nil {
+ t.Fatalf("RegisterLocal failed: %v", err)
+ }
+ } else {
+ err = r.RegisterRemote(config.RemoteAgentConfig{
+ ID: id,
+ Name: id,
+ Address: address,
+ })
+ if err != nil {
+ t.Fatalf("RegisterRemote failed: %v", err)
+ }
+ }
+
+ // Verify health
+ if tt.agentType == "remote" && tt.enabled {
+ // Must wait for health check
+ timeout := time.After(2 * time.Second)
+ ticker := time.NewTicker(20 * time.Millisecond)
+ defer ticker.Stop()
+
+ success := false
+ for !success {
+ select {
+ case <-timeout:
+ t.Fatal("timed out waiting for agent to become healthy")
+ case <-ticker.C:
+ info, err := r.GetInfo(id)
+ if err != nil {
+ t.Fatalf("GetInfo failed: %v", err)
+ }
+ // Remote agents might start unhealthy until checked
+ if info.Healthy {
+ success = true
+ }
+ }
+ }
+ } else {
+ // Should be immediately healthy (local or disabled)
+ // Important: for disabled+remote, it is optimistically healthy.
+ // For local, it is always healthy (unless manually updated, but default is true).
+
+ // Wait a tiny bit just in case async things happen, but usually sync for registration
+ time.Sleep(10 * time.Millisecond)
+
+ info, err := r.GetInfo(id)
+ if err != nil {
+ t.Fatalf("GetInfo failed: %v", err)
+ }
+ if info.Healthy != tt.expectHealthy {
+ t.Errorf("expected healthy=%v, got %v", tt.expectHealthy, info.Healthy)
+ }
+ }
+ })
+ }
+}
+
+
+
+func TestRegistry_GracefulShutdown(t *testing.T) {
+ healthCheckConfig := config.HealthCheckConfig{
+ Enabled: true,
+ Interval: 10 * time.Millisecond,
+ }
+ r, regErr := NewRegistry(healthCheckConfig)
+ if regErr != nil {
+ t.Fatalf("NewRegistry failed: %v", regErr)
+ }
+
+ // Register multiple agents to create workload
+ for i := 0; i < 50; i++ {
+ _ = r.RegisterRemote(config.RemoteAgentConfig{
+ ID: "remote-shutdown-test-" + string(rune(i)),
+ Name: "Shutdown Test Remote",
+ Address: "localhost:1234",
+ })
+ }
+
+ // Let it run for a bit to ensure performChecks triggers
+ time.Sleep(50 * time.Millisecond)
+
+ // Close should return specific errors for failed agents, but NOT panic or deadlock
+ // We are testing for absence of panic/deadlock here.
+ _ = r.Close()
+}