Agent Executor (AX) is Google's open agentic orchestration runtime
Go
allow disable active health check + encapsulate health monitor into a separate class (#15)
JoyceLiu7a855eb7654a
diff --git a/cmd/gar/serve.go b/cmd/gar/serve.goindex 451aeeb..7541b73 100644--- a/cmd/gar/serve.go+++ b/cmd/gar/serve.go@@ -112,10 +112,10 @@ func newControllerFromConfig(ctx context.Context, cfg *config.Config) (*controll// Build controller configcontrollerConfig := controller.Config{- EventLogFactory: eventLogFactory,- PlannerFactory: plannerFactory,- MaxSteps: cfg.MaxSteps,- HealthCheckInterval: cfg.HealthCheck.Interval,+ EventLogFactory: eventLogFactory,+ PlannerFactory: plannerFactory,+ MaxSteps: cfg.MaxSteps,+ HealthCheck: cfg.HealthCheck,}// Create controllerdiff --git a/examples/local_agent/main.go b/examples/local_agent/main.goindex b3039c1..3bdd38d 100644--- a/examples/local_agent/main.go+++ b/examples/local_agent/main.go@@ -47,8 +47,11 @@ func main() {}c, err := controller.New(ctx, controller.Config{- MaxSteps: 10,- HealthCheckInterval: 30 * time.Second,+ MaxSteps: 10,+ HealthCheck: config.HealthCheckConfig{+ Enabled: true,+ Interval: 30 * time.Second,+ },})if err != nil {log.Fatalf("Error creating controller: %v\n", err)diff --git a/gar.yaml b/gar.yamlindex 5b4277e..eac0d4a 100644--- a/gar.yaml+++ b/gar.yaml@@ -26,7 +26,9 @@ eventlog:max_steps: 50health_check:- # Health check interval for agents (e.g., "30s", "1m")+ # Whether to enable active health checking+ enabled: true+ # Active health check interval for agents (e.g., "30s", "1m")interval: 30s# Planner configuration (optional)diff --git a/internal/config/config.go b/internal/config/config.goindex b51b175..d9c618b 100644--- a/internal/config/config.go+++ b/internal/config/config.go@@ -34,9 +34,14 @@ type Config struct {RemoteAgents []RemoteAgentConfig `yaml:"remote_agents,omitempty"` // List of remote agents to register}-// PlannerConfig configures the planner.-type PlannerConfig struct {- Gemini GeminiPlannerConfig `yaml:"gemini"`+// HealthCheckConfig defines the configuration for agent health checks.+// When enabled, the controller will perform active polling to check the health of registered agents and+// flip the health status of agents that are not responsive. When disabled, the controller will not perform+// any health checks and will assume all agents are healthy.+// TODO(lhuan): Add passive health checks and discovery rules.+type HealthCheckConfig struct {+ Enabled bool `yaml:"enabled"` // default: false (no active polling)+ Interval time.Duration `yaml:"interval"` // default: 30s}// ServerConfig configures the gRPC server.@@ -49,8 +54,9 @@ type EventLogConfig struct {Dir string `yaml:"dir"` // Directory for event log files}-type HealthCheckConfig struct {- Interval time.Duration `yaml:"interval"` // Interval between health checks+// PlannerConfig configures the planner.+type PlannerConfig struct {+ Gemini GeminiPlannerConfig `yaml:"gemini"`}// GeminiPlannerConfig configures the Gemini-based planner.@@ -116,8 +122,8 @@ func (c *Config) setDefaults() {if c.MaxSteps == 0 {c.MaxSteps = 100}-- if c.HealthCheck.Interval == 0 {+ // HealthCheck defaults+ if c.HealthCheck.Enabled && c.HealthCheck.Interval == 0 {c.HealthCheck.Interval = 30 * time.Second}}@@ -133,8 +139,13 @@ func (c *Config) Validate() error {if c.MaxSteps <= 0 {return fmt.Errorf("max_steps must be positive")}- if c.HealthCheck.Interval <= 0 {- return fmt.Errorf("health_check.interval must be positive")++ // Validate health check+ if c.HealthCheck.Enabled {+ if c.HealthCheck.Interval <= 0 {+ return fmt.Errorf("health_check.interval must be positive when enabled")+ }}+return nil}diff --git a/internal/controller/controller.go b/internal/controller/controller.goindex 5dc894e..81ab44b 100644--- a/internal/controller/controller.go+++ b/internal/controller/controller.go@@ -20,9 +20,9 @@ import ("context""fmt""sync"- "time""github.com/google/gar/agent"+ "github.com/google/gar/internal/config""github.com/google/gar/internal/eventlog""github.com/google/gar/proto")@@ -32,7 +32,6 @@ import (type Controller struct {inFlightSessionsMu sync.MutexinFlightSessions map[string]struct{}-sessionManager *SessionManagerregistry *RegistryloopExecutor *LoopExecutor@@ -46,8 +45,8 @@ type Config struct {EventLogFactory eventlog.EventLogFactoryPlannerFactory PlannerFactory// TODO(jbd): Add CompactionFunc.- HealthCheckInterval time.Duration- MaxSteps int+ HealthCheck config.HealthCheckConfig+ MaxSteps int}// New creates a new controller instance.@@ -55,9 +54,7 @@ func New(ctx context.Context, config Config) (*Controller, error) {if config.MaxSteps == 0 {config.MaxSteps = 100}- if config.HealthCheckInterval == 0 {- config.HealthCheckInterval = 30 * time.Second- }+if config.EventLogFactory == nil {config.EventLogFactory = func(sessionID string) (eventlog.EventLog, error) {return eventlog.NewFileEventLog(eventlog.FileConfig{@@ -71,7 +68,10 @@ func New(ctx context.Context, config Config) (*Controller, error) {sessionManager := NewSessionManager(config.EventLogFactory)// Initialize agent registry- registry := NewRegistry(config.HealthCheckInterval)+ registry, err := NewRegistry(config.HealthCheck)+ if err != nil {+ return nil, fmt.Errorf("failed to create registry: %w", err)+ }// Determine plan function// If no planner factory is provided, use the default Gemini planner.diff --git a/internal/controller/gemini.go b/internal/controller/gemini.goindex f7555ef..2170d78 100644--- a/internal/controller/gemini.go+++ b/internal/controller/gemini.go@@ -140,6 +140,7 @@ func agentsToTools(registry *Registry) ([]*genai.Tool, error) {}var tools []*genai.Tool+ // TODO(lhuan): Check if agentsToTools returns an error or empty list and return a friendly "no agent available, try later" error.for _, id := range healthyAgents {info, err := registry.GetInfo(id)if err != nil {diff --git a/internal/controller/health.go b/internal/controller/health.gonew file mode 100644index 0000000..80e92ad--- /dev/null+++ b/internal/controller/health.go@@ -0,0 +1,78 @@+package controller++import (+ "sync"+ "time"++ "github.com/google/gar/internal/config"+)++// HealthMonitor manages the periodic health checking of agents.+type HealthMonitor struct {+ config config.HealthCheckConfig+ registry *Registry+ ticker *time.Ticker+ stop chan struct{}+ wg sync.WaitGroup+}++// NewHealthMonitor creates a new health monitor.+func NewHealthMonitor(cfg config.HealthCheckConfig, r *Registry) *HealthMonitor {+ return &HealthMonitor{+ config: cfg,+ registry: r,+ stop: make(chan struct{}),+ }+}++func (hm *HealthMonitor) Start() {+ interval := hm.config.Interval+ hm.ticker = time.NewTicker(interval)+ hm.wg.Add(1)+ go hm.run()+}++// Stop halts the health check loop.+func (hm *HealthMonitor) Stop() {+ if hm.ticker != nil {+ hm.ticker.Stop()+ close(hm.stop)+ hm.wg.Wait()+ }+}++// run is the main loop for health checks.+func (hm *HealthMonitor) run() {+ defer hm.wg.Done()++ for {+ select {+ case <-hm.stop:+ return+ case <-hm.ticker.C:+ hm.performChecks()+ }+ }+}++// performChecks triggers health checks for all agents.+func (hm *HealthMonitor) performChecks() {+ // Get IDs safely+ hm.registry.mu.RLock()+ ids := make([]string, 0, len(hm.registry.agents))+ for id := range hm.registry.agents {+ ids = append(ids, id)+ }+ hm.registry.mu.RUnlock()++ // Run checks in parallel+ var wg sync.WaitGroup+ for _, id := range ids {+ wg.Add(1)+ go func(agentID string) {+ defer wg.Done()+ _ = hm.registry.healthCheck(agentID)+ }(id)+ }+ wg.Wait()+}diff --git a/internal/controller/loop.go b/internal/controller/loop.goindex fb4c90c..9fd174f 100644--- a/internal/controller/loop.go+++ b/internal/controller/loop.go@@ -145,6 +145,8 @@ func (e *LoopExecutor) runTask(ctx context.Context, session *Session, task *Taskreturn fmt.Errorf("failed to get agent: %w", err)}+ // TODO(lhuan): Handle scenario where agent is marked healthy (optimistic) but fails to respond (e.g. still starting up).+ // Return "agent internal error, try again later" to the user.if err := ag.Process(ctx, session.ID(), &proto.ProcessRequest{Contents: task.Inputs,}, taskOutputHandler); err != nil {diff --git a/internal/controller/registry.go b/internal/controller/registry.goindex 8122e14..3ac5102 100644--- a/internal/controller/registry.go+++ b/internal/controller/registry.go@@ -46,32 +46,28 @@ type AgentInfo struct {// Registry manages a collection of local and remote agents.// It provides agent discovery, health monitoring, and load balancing.type Registry struct {- mu sync.RWMutex- agents map[string]agent.Agent- agentInfo map[string]*AgentInfo- healthCheckTicker *time.Ticker- stopHealthCheck chan struct{}+ mu sync.RWMutex+ agents map[string]agent.Agent+ agentInfo map[string]*AgentInfo+ monitor *HealthMonitor}// NewRegistry creates a new agent registry.-func NewRegistry(healthCheckInterval time.Duration) *Registry {- if healthCheckInterval == 0 {- healthCheckInterval = 30 * time.Second+func NewRegistry(healthCheckConfig config.HealthCheckConfig) (*Registry, error) {+ if healthCheckConfig.Enabled && healthCheckConfig.Interval <= 0 {+ return nil, fmt.Errorf("invalid health check interval: %v", healthCheckConfig.Interval)}r := &Registry{- agents: make(map[string]agent.Agent),- agentInfo: make(map[string]*AgentInfo),- stopHealthCheck: make(chan struct{}),+ agents: make(map[string]agent.Agent),+ agentInfo: make(map[string]*AgentInfo),}-- // Start background health check if interval is positive- if healthCheckInterval > 0 {- r.healthCheckTicker = time.NewTicker(healthCheckInterval)- go r.runHealthChecks()+ if healthCheckConfig.Enabled {+ r.monitor = NewHealthMonitor(healthCheckConfig, r)+ r.monitor.Start()}- return r+ return r, nil}// RegisterLocal registers a local (in-process) agent.@@ -102,6 +98,8 @@ func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) error {r.mu.Lock()defer r.mu.Unlock()+ // TODO(lhuan): Consider enforcing health check during registration. Only allow registration if the agent is reachable and healthy.+if _, ok := r.agents[cfg.ID]; ok {return fmt.Errorf("agent %s already registered", cfg.ID)}@@ -122,7 +120,7 @@ func (r *Registry) RegisterRemote(cfg config.RemoteAgentConfig) error {Name: cfg.Name,Description: cfg.Description,Type: AgentTypeRemote,- Healthy: false, // Will be checked by health monitor+ Healthy: (r.monitor == nil), // Optimistic if no health check, pessimistic if health check is enabledLastHealthCheck: time.Time{},Metadata: cfg.Metadata,}@@ -169,8 +167,8 @@ func (r *Registry) GetInfo(id string) (*AgentInfo, error) {r.mu.RLock()defer r.mu.RUnlock()- info, exists := r.agentInfo[id]- if !exists {+ info, ok := r.agentInfo[id]+ if !ok {return nil, fmt.Errorf("agent %s not found", id)}@@ -205,6 +203,7 @@ func (r *Registry) ListHealthy() []string {return ids}+// healthCheck performs a health check on a specific agent.func (r *Registry) healthCheck(id string) error {a, err := r.Get(id)@@ -229,38 +228,18 @@ func (r *Registry) healthCheck(id string) error {return err}-// runHealthChecks runs periodic health checks on all agents.-func (r *Registry) runHealthChecks() {- for {- select {- case <-r.healthCheckTicker.C:- r.performHealthChecks()- case <-r.stopHealthCheck:- return- }- }-}--// performHealthChecks checks the health of all registered agents.-func (r *Registry) performHealthChecks() {- ids := r.List()- for _, id := range ids {- // Run health check (ignore errors, status is updated in HealthCheck method)- _ = r.healthCheck(id)- }-}// Close stops the registry and closes all agents.func (r *Registry) Close() error {+ // Stop health checks before acquiring the lock to avoid deadlock+ // (monitor.Stop waits for background routines that might need the lock)+ if r.monitor != nil {+ r.monitor.Stop()+ }+r.mu.Lock()defer r.mu.Unlock()- // Stop health checks- if r.healthCheckTicker != nil {- r.healthCheckTicker.Stop()- close(r.stopHealthCheck)- }-// Close all agentsvar firstErr errorfor id, a := range r.agents {diff --git a/internal/controller/registry_test.go b/internal/controller/registry_test.gonew file mode 100644index 0000000..d9287c8--- /dev/null+++ b/internal/controller/registry_test.go@@ -0,0 +1,232 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controller++// TODO(lhuan): Setup a better automated testing framework++import (+ "context"+ "net"+ "testing"+ "time"++ "google.golang.org/grpc"++ "github.com/google/gar/agent"+ "github.com/google/gar/internal/config"+ "github.com/google/gar/proto"+)++// MockAgent is a mock implementation of the Agent interface for testing.+type MockAgent struct {+ processFunc func(ctx context.Context, sessionID string, req *proto.ProcessRequest, handler agent.OutputHandler) error+ healthCheckFunc func(ctx context.Context) error+ closeFunc func() error+}++func (m *MockAgent) Process(ctx context.Context, sessionID string, req *proto.ProcessRequest, handler agent.OutputHandler) error {+ if m.processFunc != nil {+ return m.processFunc(ctx, sessionID, req, handler)+ }+ return nil+}++func (m *MockAgent) HealthCheck(ctx context.Context) error {+ if m.healthCheckFunc != nil {+ return m.healthCheckFunc(ctx)+ }+ return nil+}++func (m *MockAgent) Close() error {+ if m.closeFunc != nil {+ return m.closeFunc()+ }+ return nil+}++type mockAgentServer struct {+ proto.UnimplementedAgentServiceServer+ healthy bool+}++func (s *mockAgentServer) HealthCheck(ctx context.Context, req *proto.HealthCheckRequest) (*proto.HealthCheckResponse, error) {+ return &proto.HealthCheckResponse{Healthy: s.healthy}, nil+}++func startMockGRPCServer(t *testing.T, healthy bool) (string, func()) {+ lis, err := net.Listen("tcp", "localhost:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ s := grpc.NewServer()+ proto.RegisterAgentServiceServer(s, &mockAgentServer{healthy: healthy})+ go func() {+ if err := s.Serve(lis); err != nil {+ // server might be closed+ }+ }()+ return lis.Addr().String(), func() {+ s.Stop()+ lis.Close()+ }+}++++func TestRegistry_HealthCheckScenarios(t *testing.T) {+ tests := []struct {+ name string+ enabled bool+ agentType string+ expectHealthy bool+ }{+ {+ name: "Local Agent (Always Healthy)",+ enabled: true,+ agentType: "local",+ expectHealthy: true,+ },+ {+ name: "Local Agent - Check Disabled (Always Healthy)",+ enabled: false,+ agentType: "local",+ expectHealthy: true,+ },+ {+ name: "Remote Agent - Check Enabled (Eventually Healthy)",+ enabled: true,+ agentType: "remote",+ expectHealthy: true, // Eventually true+ },+ {+ name: "Remote Agent - Check Disabled (Optimistically Healthy)",+ enabled: false,+ agentType: "remote",+ expectHealthy: true, // Optimistically true+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ // Mock server for remote agent+ address, cleanup := startMockGRPCServer(t, true)+ defer cleanup()++ cfg := config.HealthCheckConfig{+ Enabled: tt.enabled,+ Interval: 10 * time.Millisecond,+ }+ r, err := NewRegistry(cfg)+ if err != nil {+ t.Fatalf("NewRegistry failed: %v", err)+ }+ defer r.Close()++ id := "test-agent"++ if tt.agentType == "local" {+ // Mock local agent+ validMockAgent := &MockAgent{}+ err = r.RegisterLocal(config.LocalAgentConfig{+ ID: id,+ Name: id,+ Agent: validMockAgent,+ })+ if err != nil {+ t.Fatalf("RegisterLocal failed: %v", err)+ }+ } else {+ err = r.RegisterRemote(config.RemoteAgentConfig{+ ID: id,+ Name: id,+ Address: address,+ })+ if err != nil {+ t.Fatalf("RegisterRemote failed: %v", err)+ }+ }++ // Verify health+ if tt.agentType == "remote" && tt.enabled {+ // Must wait for health check+ timeout := time.After(2 * time.Second)+ ticker := time.NewTicker(20 * time.Millisecond)+ defer ticker.Stop()++ success := false+ for !success {+ select {+ case <-timeout:+ t.Fatal("timed out waiting for agent to become healthy")+ case <-ticker.C:+ info, err := r.GetInfo(id)+ if err != nil {+ t.Fatalf("GetInfo failed: %v", err)+ }+ // Remote agents might start unhealthy until checked+ if info.Healthy {+ success = true+ }+ }+ }+ } else {+ // Should be immediately healthy (local or disabled)+ // Important: for disabled+remote, it is optimistically healthy.+ // For local, it is always healthy (unless manually updated, but default is true).++ // Wait a tiny bit just in case async things happen, but usually sync for registration+ time.Sleep(10 * time.Millisecond)++ info, err := r.GetInfo(id)+ if err != nil {+ t.Fatalf("GetInfo failed: %v", err)+ }+ if info.Healthy != tt.expectHealthy {+ t.Errorf("expected healthy=%v, got %v", tt.expectHealthy, info.Healthy)+ }+ }+ })+ }+}++++func TestRegistry_GracefulShutdown(t *testing.T) {+ healthCheckConfig := config.HealthCheckConfig{+ Enabled: true,+ Interval: 10 * time.Millisecond,+ }+ r, regErr := NewRegistry(healthCheckConfig)+ if regErr != nil {+ t.Fatalf("NewRegistry failed: %v", regErr)+ }++ // Register multiple agents to create workload+ for i := 0; i < 50; i++ {+ _ = r.RegisterRemote(config.RemoteAgentConfig{+ ID: "remote-shutdown-test-" + string(rune(i)),+ Name: "Shutdown Test Remote",+ Address: "localhost:1234",+ })+ }++ // Let it run for a bit to ensure performChecks triggers+ time.Sleep(50 * time.Millisecond)++ // Close should return specific errors for failed agents, but NOT panic or deadlock+ // We are testing for absence of panic/deadlock here.+ _ = r.Close()+}