Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

Add health check for harness server. (#72)

Junjie Wang6abe1d4a2805
diff --git a/cmd/ax/harness.go b/cmd/ax/harness.go
index 3faf6c9..89a5858 100644
--- a/cmd/ax/harness.go
+++ b/cmd/ax/harness.go
@@ -30,6 +30,8 @@ import (
"github.com/google/ax/proto"
"github.com/spf13/cobra"
"google.golang.org/grpc"
+ "google.golang.org/grpc/health"
+ "google.golang.org/grpc/health/grpc_health_v1"
)
var (
@@ -59,6 +61,11 @@ func runHarness(cmd *cobra.Command, args []string) error {
harnessServer := NewHarnessServiceServer()
proto.RegisterHarnessServiceServer(grpcServer, harnessServer)
+ // Serve the standard gRPC health protocol.
+ healthServer := health.NewServer()
+ healthServer.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
+ grpc_health_v1.RegisterHealthServer(grpcServer, healthServer)
+
// Graceful shutdown handling
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
diff --git a/internal/harness/substrate.go b/internal/harness/substrate.go
index a320915..fe61f40 100644
--- a/internal/harness/substrate.go
+++ b/internal/harness/substrate.go
@@ -28,6 +28,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/status"
"github.com/google/ax/internal/experimental/k8s/ate"
@@ -35,6 +36,10 @@ import (
"github.com/google/uuid"
)
+// healthCheckTimeout defines the maximum time Start waits for a freshly
+// created/resumed actor's harness to become reachable and ready.
+const healthCheckTimeout = 60 * time.Second
+
// SubstrateHarness manages execution in a SubstrATE sandboxed actor over gRPC HarnessService.
type SubstrateHarness struct {
harnessID string
@@ -95,13 +100,20 @@ func (h *SubstrateHarness) Start(ctx context.Context, conversationID string) (Ex
return nil, fmt.Errorf("actor %s has no active worker IP address", conversationID)
}
- // 2. Establish connection to the actor's worker IP
+ // Establish connection to the actor's worker IP
workerAddr := fmt.Sprintf("%s:%d", actor.AteomPodIp, h.port)
conn, err := grpc.NewClient(workerAddr, h.dialOpts...)
if err != nil {
return nil, fmt.Errorf("failed to dial remote harness service at %s: %w", workerAddr, err)
}
+ // Wait for the harness to be reachable and ready before handing back the
+ // execution.
+ if err := waitForHealthy(ctx, conn, healthCheckTimeout); err != nil {
+ conn.Close()
+ return nil, fmt.Errorf("harness for %s not ready at %s: %w", conversationID, workerAddr, err)
+ }
+
return &substrateExecution{
harness: h,
conversationID: conversationID,
@@ -111,6 +123,42 @@ func (h *SubstrateHarness) Start(ctx context.Context, conversationID string) (Ex
}, nil
}
+// waitForHealthy blocks until the harness behind conn reports SERVING via the
+// standard gRPC health protocol until timeout. A harness that is reachable
+// but does not implement the health service (Unimplemented) is treated as
+// ready; connection failures (Unavailable) and NOT_SERVING are retried.
+func waitForHealthy(ctx context.Context, conn *grpc.ClientConn, timeout time.Duration) error {
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+
+ client := grpc_health_v1.NewHealthClient(conn)
+ const maxBackoff = 2 * time.Second
+ backoff := 100 * time.Millisecond
+ for {
+ resp, err := client.Check(ctx, &grpc_health_v1.HealthCheckRequest{Service: ""})
+ if err == nil && resp.GetStatus() == grpc_health_v1.HealthCheckResponse_SERVING {
+ return nil
+ }
+ if status.Code(err) == codes.Unimplemented {
+ // Reachable but no health service: the port is up, proceed.
+ return nil
+ }
+
+ select {
+ case <-ctx.Done():
+ if err != nil {
+ return fmt.Errorf("harness not healthy within %s: %w", timeout, err)
+ }
+ return fmt.Errorf("harness not healthy within %s (last status: %s)", timeout, resp.GetStatus())
+ case <-time.After(backoff):
+ }
+ backoff *= 2
+ if backoff > maxBackoff {
+ backoff = maxBackoff
+ }
+ }
+}
+
type substrateExecution struct {
harness *SubstrateHarness
conversationID string
diff --git a/internal/harness/substrate_test.go b/internal/harness/substrate_test.go
new file mode 100644
index 0000000..0ec3d3b
--- /dev/null
+++ b/internal/harness/substrate_test.go
@@ -0,0 +1,115 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package harness
+
+import (
+ "context"
+ "net"
+ "testing"
+ "time"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/health"
+ "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+// startHealthTestServer starts a gRPC server on a random local port. If hs is
+// non-nil the standard health service is registered. Returns the listen address.
+func startHealthTestServer(t *testing.T, hs *health.Server) string {
+ t.Helper()
+ lis, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ t.Fatalf("failed to listen: %v", err)
+ }
+ s := grpc.NewServer()
+ if hs != nil {
+ grpc_health_v1.RegisterHealthServer(s, hs)
+ }
+ go func() {
+ _ = s.Serve(lis)
+ }()
+ t.Cleanup(s.Stop)
+ return lis.Addr().String()
+}
+
+func dialTestConn(t *testing.T, addr string) *grpc.ClientConn {
+ t.Helper()
+ conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ t.Fatalf("failed to dial %s: %v", addr, err)
+ }
+ t.Cleanup(func() { conn.Close() })
+ return conn
+}
+
+func TestWaitForHealthy_Serving(t *testing.T) {
+ hs := health.NewServer()
+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
+ conn := dialTestConn(t, startHealthTestServer(t, hs))
+
+ if err := waitForHealthy(context.Background(), conn, 5*time.Second); err != nil {
+ t.Fatalf("expected healthy, got %v", err)
+ }
+}
+
+func TestWaitForHealthy_UnimplementedProceeds(t *testing.T) {
+ // Server is up but does not register the health service.
+ conn := dialTestConn(t, startHealthTestServer(t, nil))
+
+ if err := waitForHealthy(context.Background(), conn, 5*time.Second); err != nil {
+ t.Fatalf("expected to proceed when health is unimplemented, got %v", err)
+ }
+}
+
+func TestWaitForHealthy_TimesOut(t *testing.T) {
+ hs := health.NewServer()
+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING)
+ conn := dialTestConn(t, startHealthTestServer(t, hs))
+
+ if err := waitForHealthy(context.Background(), conn, 500*time.Millisecond); err == nil {
+ t.Fatal("expected timeout error while NOT_SERVING, got nil")
+ }
+}
+
+func TestWaitForHealthy_StatusChange(t *testing.T) {
+ hs := health.NewServer()
+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING)
+ conn := dialTestConn(t, startHealthTestServer(t, hs))
+
+ go func() {
+ time.Sleep(150 * time.Millisecond)
+ hs.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
+ }()
+
+ if err := waitForHealthy(context.Background(), conn, 5*time.Second); err != nil {
+ t.Fatalf("expected healthy after status flip, got %v", err)
+ }
+}
+
+func TestWaitForHealthy_ServerDown(t *testing.T) {
+ // Reserve a port then release it so nothing is listening there.
+ lis, err := net.Listen("tcp", "localhost:0")
+ if err != nil {
+ t.Fatalf("failed to listen: %v", err)
+ }
+ addr := lis.Addr().String()
+ lis.Close()
+ conn := dialTestConn(t, addr)
+
+ if err := waitForHealthy(context.Background(), conn, 500*time.Millisecond); err == nil {
+ t.Fatal("expected timeout error when server is down, got nil")
+ }
+}
diff --git a/python/antigravity/harness_server.py b/python/antigravity/harness_server.py
index ae34a7b..8f7513f 100644
--- a/python/antigravity/harness_server.py
+++ b/python/antigravity/harness_server.py
@@ -24,6 +24,7 @@ import importlib.util
import logging
import sys
import grpc
+from grpc_health.v1 import health, health_pb2, health_pb2_grpc
from google.protobuf.struct_pb2 import Struct
from python.proto import ax_pb2
@@ -210,6 +211,11 @@ class AntigravityHarnessServiceServicer(ax_pb2_grpc.HarnessServiceServicer):
async def serve(host: str, port: int):
server = grpc.aio.server()
ax_pb2_grpc.add_HarnessServiceServicer_to_server(AntigravityHarnessServiceServicer(), server)
+
+ # Serve the standard gRPC health protocol.
+ health_servicer = health.aio.HealthServicer()
+ health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
+ await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING)
listen_addr = f"{host}:{port}"
server.add_insecure_port(listen_addr)
diff --git a/python/antigravity/harness_server_test.py b/python/antigravity/harness_server_test.py
index 8b4baf3..ae14f6c 100644
--- a/python/antigravity/harness_server_test.py
+++ b/python/antigravity/harness_server_test.py
@@ -93,3 +93,25 @@ def test_grpc_connect_success(mock_config, monkeypatch):
await server.stop(0)
asyncio.run(_run())
+
+
+def test_health_check():
+ async def _run():
+ from grpc_health.v1 import health, health_pb2, health_pb2_grpc
+
+ server = grpc.aio.server()
+ ax_pb2_grpc.add_HarnessServiceServicer_to_server(AntigravityHarnessServiceServicer(), server)
+ health_servicer = health.aio.HealthServicer()
+ health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
+ await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING)
+ port = server.add_insecure_port("localhost:0")
+ await server.start()
+ try:
+ async with grpc.aio.insecure_channel(f"localhost:{port}") as channel:
+ stub = health_pb2_grpc.HealthStub(channel)
+ resp = await stub.Check(health_pb2.HealthCheckRequest(service=""))
+ assert resp.status == health_pb2.HealthCheckResponse.SERVING
+ finally:
+ await server.stop(0)
+
+ asyncio.run(_run())
diff --git a/python/antigravity/requirements.txt b/python/antigravity/requirements.txt
new file mode 100644
index 0000000..dd9e44e
--- /dev/null
+++ b/python/antigravity/requirements.txt
@@ -0,0 +1,9 @@
+# Runtime dependencies for the antigravity HarnessService server
+# (python/antigravity/harness_server.py).
+#
+# grpcio-health-checking is published on public PyPI but not the internal
+# mirror, so allow PyPI as an extra index for it.
+--extra-index-url https://pypi.org/simple/
+
+google-antigravity
+grpcio-health-checking==1.81.0