Agent Executor (AX) is Google's open agentic orchestration runtime
Go
Add health check for harness server. (#72)
Junjie Wang6abe1d4a2805
diff --git a/cmd/ax/harness.go b/cmd/ax/harness.goindex 3faf6c9..89a5858 100644--- a/cmd/ax/harness.go+++ b/cmd/ax/harness.go@@ -30,6 +30,8 @@ import ("github.com/google/ax/proto""github.com/spf13/cobra""google.golang.org/grpc"+ "google.golang.org/grpc/health"+ "google.golang.org/grpc/health/grpc_health_v1")var (@@ -59,6 +61,11 @@ func runHarness(cmd *cobra.Command, args []string) error {harnessServer := NewHarnessServiceServer()proto.RegisterHarnessServiceServer(grpcServer, harnessServer)+ // Serve the standard gRPC health protocol.+ healthServer := health.NewServer()+ healthServer.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)+ grpc_health_v1.RegisterHealthServer(grpcServer, healthServer)+// Graceful shutdown handlingsigChan := make(chan os.Signal, 1)signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)diff --git a/internal/harness/substrate.go b/internal/harness/substrate.goindex a320915..fe61f40 100644--- a/internal/harness/substrate.go+++ b/internal/harness/substrate.go@@ -28,6 +28,7 @@ import ("google.golang.org/grpc/codes""google.golang.org/grpc/credentials""google.golang.org/grpc/credentials/insecure"+ "google.golang.org/grpc/health/grpc_health_v1""google.golang.org/grpc/status""github.com/google/ax/internal/experimental/k8s/ate"@@ -35,6 +36,10 @@ import ("github.com/google/uuid")+// healthCheckTimeout defines the maximum time Start waits for a freshly+// created/resumed actor's harness to become reachable and ready.+const healthCheckTimeout = 60 * time.Second+// SubstrateHarness manages execution in a SubstrATE sandboxed actor over gRPC HarnessService.type SubstrateHarness struct {harnessID string@@ -95,13 +100,20 @@ func (h *SubstrateHarness) Start(ctx context.Context, conversationID string) (Exreturn nil, fmt.Errorf("actor %s has no active worker IP address", conversationID)}- // 2. Establish connection to the actor's worker IP+ // Establish connection to the actor's worker IPworkerAddr := fmt.Sprintf("%s:%d", actor.AteomPodIp, h.port)conn, err := grpc.NewClient(workerAddr, h.dialOpts...)if err != nil {return nil, fmt.Errorf("failed to dial remote harness service at %s: %w", workerAddr, err)}+ // Wait for the harness to be reachable and ready before handing back the+ // execution.+ if err := waitForHealthy(ctx, conn, healthCheckTimeout); err != nil {+ conn.Close()+ return nil, fmt.Errorf("harness for %s not ready at %s: %w", conversationID, workerAddr, err)+ }+return &substrateExecution{harness: h,conversationID: conversationID,@@ -111,6 +123,42 @@ func (h *SubstrateHarness) Start(ctx context.Context, conversationID string) (Ex}, nil}+// waitForHealthy blocks until the harness behind conn reports SERVING via the+// standard gRPC health protocol until timeout. A harness that is reachable+// but does not implement the health service (Unimplemented) is treated as+// ready; connection failures (Unavailable) and NOT_SERVING are retried.+func waitForHealthy(ctx context.Context, conn *grpc.ClientConn, timeout time.Duration) error {+ ctx, cancel := context.WithTimeout(ctx, timeout)+ defer cancel()++ client := grpc_health_v1.NewHealthClient(conn)+ const maxBackoff = 2 * time.Second+ backoff := 100 * time.Millisecond+ for {+ resp, err := client.Check(ctx, &grpc_health_v1.HealthCheckRequest{Service: ""})+ if err == nil && resp.GetStatus() == grpc_health_v1.HealthCheckResponse_SERVING {+ return nil+ }+ if status.Code(err) == codes.Unimplemented {+ // Reachable but no health service: the port is up, proceed.+ return nil+ }++ select {+ case <-ctx.Done():+ if err != nil {+ return fmt.Errorf("harness not healthy within %s: %w", timeout, err)+ }+ return fmt.Errorf("harness not healthy within %s (last status: %s)", timeout, resp.GetStatus())+ case <-time.After(backoff):+ }+ backoff *= 2+ if backoff > maxBackoff {+ backoff = maxBackoff+ }+ }+}+type substrateExecution struct {harness *SubstrateHarnessconversationID stringdiff --git a/internal/harness/substrate_test.go b/internal/harness/substrate_test.gonew file mode 100644index 0000000..0ec3d3b--- /dev/null+++ b/internal/harness/substrate_test.go@@ -0,0 +1,115 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package harness++import (+ "context"+ "net"+ "testing"+ "time"++ "google.golang.org/grpc"+ "google.golang.org/grpc/credentials/insecure"+ "google.golang.org/grpc/health"+ "google.golang.org/grpc/health/grpc_health_v1"+)++// startHealthTestServer starts a gRPC server on a random local port. If hs is+// non-nil the standard health service is registered. Returns the listen address.+func startHealthTestServer(t *testing.T, hs *health.Server) string {+ t.Helper()+ lis, err := net.Listen("tcp", "localhost:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ s := grpc.NewServer()+ if hs != nil {+ grpc_health_v1.RegisterHealthServer(s, hs)+ }+ go func() {+ _ = s.Serve(lis)+ }()+ t.Cleanup(s.Stop)+ return lis.Addr().String()+}++func dialTestConn(t *testing.T, addr string) *grpc.ClientConn {+ t.Helper()+ conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))+ if err != nil {+ t.Fatalf("failed to dial %s: %v", addr, err)+ }+ t.Cleanup(func() { conn.Close() })+ return conn+}++func TestWaitForHealthy_Serving(t *testing.T) {+ hs := health.NewServer()+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)+ conn := dialTestConn(t, startHealthTestServer(t, hs))++ if err := waitForHealthy(context.Background(), conn, 5*time.Second); err != nil {+ t.Fatalf("expected healthy, got %v", err)+ }+}++func TestWaitForHealthy_UnimplementedProceeds(t *testing.T) {+ // Server is up but does not register the health service.+ conn := dialTestConn(t, startHealthTestServer(t, nil))++ if err := waitForHealthy(context.Background(), conn, 5*time.Second); err != nil {+ t.Fatalf("expected to proceed when health is unimplemented, got %v", err)+ }+}++func TestWaitForHealthy_TimesOut(t *testing.T) {+ hs := health.NewServer()+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING)+ conn := dialTestConn(t, startHealthTestServer(t, hs))++ if err := waitForHealthy(context.Background(), conn, 500*time.Millisecond); err == nil {+ t.Fatal("expected timeout error while NOT_SERVING, got nil")+ }+}++func TestWaitForHealthy_StatusChange(t *testing.T) {+ hs := health.NewServer()+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING)+ conn := dialTestConn(t, startHealthTestServer(t, hs))++ go func() {+ time.Sleep(150 * time.Millisecond)+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)+ }()++ if err := waitForHealthy(context.Background(), conn, 5*time.Second); err != nil {+ t.Fatalf("expected healthy after status flip, got %v", err)+ }+}++func TestWaitForHealthy_ServerDown(t *testing.T) {+ // Reserve a port then release it so nothing is listening there.+ lis, err := net.Listen("tcp", "localhost:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ addr := lis.Addr().String()+ lis.Close()+ conn := dialTestConn(t, addr)++ if err := waitForHealthy(context.Background(), conn, 500*time.Millisecond); err == nil {+ t.Fatal("expected timeout error when server is down, got nil")+ }+}diff --git a/python/antigravity/harness_server.py b/python/antigravity/harness_server.pyindex ae34a7b..8f7513f 100644--- a/python/antigravity/harness_server.py+++ b/python/antigravity/harness_server.py@@ -24,6 +24,7 @@ import importlib.utilimport loggingimport sysimport grpc+from grpc_health.v1 import health, health_pb2, health_pb2_grpcfrom google.protobuf.struct_pb2 import Structfrom python.proto import ax_pb2@@ -210,6 +211,11 @@ class AntigravityHarnessServiceServicer(ax_pb2_grpc.HarnessServiceServicer):async def serve(host: str, port: int):server = grpc.aio.server()ax_pb2_grpc.add_HarnessServiceServicer_to_server(AntigravityHarnessServiceServicer(), server)++ # Serve the standard gRPC health protocol.+ health_servicer = health.aio.HealthServicer()+ health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)+ await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING)listen_addr = f"{host}:{port}"server.add_insecure_port(listen_addr)diff --git a/python/antigravity/harness_server_test.py b/python/antigravity/harness_server_test.pyindex 8b4baf3..ae14f6c 100644--- a/python/antigravity/harness_server_test.py+++ b/python/antigravity/harness_server_test.py@@ -93,3 +93,25 @@ def test_grpc_connect_success(mock_config, monkeypatch):await server.stop(0)asyncio.run(_run())+++def test_health_check():+ async def _run():+ from grpc_health.v1 import health, health_pb2, health_pb2_grpc++ server = grpc.aio.server()+ ax_pb2_grpc.add_HarnessServiceServicer_to_server(AntigravityHarnessServiceServicer(), server)+ health_servicer = health.aio.HealthServicer()+ health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)+ await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING)+ port = server.add_insecure_port("localhost:0")+ await server.start()+ try:+ async with grpc.aio.insecure_channel(f"localhost:{port}") as channel:+ stub = health_pb2_grpc.HealthStub(channel)+ resp = await stub.Check(health_pb2.HealthCheckRequest(service=""))+ assert resp.status == health_pb2.HealthCheckResponse.SERVING+ finally:+ await server.stop(0)++ asyncio.run(_run())diff --git a/python/antigravity/requirements.txt b/python/antigravity/requirements.txtnew file mode 100644index 0000000..dd9e44e--- /dev/null+++ b/python/antigravity/requirements.txt@@ -0,0 +1,9 @@+# Runtime dependencies for the antigravity HarnessService server+# (python/antigravity/harness_server.py).+#+# grpcio-health-checking is published on public PyPI but not the internal+# mirror, so allow PyPI as an extra index for it.+--extra-index-url https://pypi.org/simple/++google-antigravity+grpcio-health-checking==1.81.0