Agent Executor (AX) is Google's open agentic orchestration runtime
Go
Serve /readyz from ax harness server. (#254)
Junjie Wang948ca05b5233
diff --git a/cmd/ax/harness.go b/cmd/ax/harness.goindex cf6470a..a046c77 100644--- a/cmd/ax/harness.go+++ b/cmd/ax/harness.go@@ -18,15 +18,22 @@package mainimport (+ "context""fmt""log"+ "net"+ "net/http""os""os/exec""os/signal""strconv""syscall"+ "time""github.com/spf13/cobra"+ "google.golang.org/grpc"+ "google.golang.org/grpc/credentials/insecure"+ healthpb "google.golang.org/grpc/health/grpc_health_v1")var (@@ -34,6 +41,9 @@ var (harnessHost string)+// harnessReadyzPort is the port for the HTTP /readyz readiness endpoint.+const harnessReadyzPort = 8081+var harnessCmd = &cobra.Command{Use: "harness",Short: "Run the harness gRPC server",@@ -84,6 +94,10 @@ func runHarness(cmd *cobra.Command, args []string) error {}log.Printf("forked antigravity harness server (pid %d) on %s:%d", py.Process.Pid, harnessHost, harnessPort)+ // Serve the /readyz endpoint that substrate's readiness probe polls (during+ // golden snapshotting and per-actor Run/Restore).+ go serveReadyz(harnessReadyzPort, harnessPort)+// Forward termination signals to the child so substrate can stop the actor.sigChan := make(chan os.Signal, 1)signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)@@ -98,3 +112,36 @@ func runHarness(cmd *cobra.Command, args []string) error {}return nil}++// serveReadyz serves the HTTP /readyz endpoint on readyzPort that substrate's+// readiness probe polls (during golden snapshotting and per-actor Run/Restore).+// Each request forwards to the forked harness's gRPC health Check.+func serveReadyz(readyzPort, grpcPort int) {+ conn, err := grpc.NewClient(+ net.JoinHostPort("127.0.0.1", strconv.Itoa(grpcPort)),+ grpc.WithTransportCredentials(insecure.NewCredentials()),+ )+ if err != nil {+ log.Printf("readyz: failed to create gRPC health client: %v", err)+ return+ }+ healthClient := healthpb.NewHealthClient(conn)++ mux := http.NewServeMux()+ mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {+ ctx, cancel := context.WithTimeout(r.Context(), time.Second)+ defer cancel()+ resp, err := healthClient.Check(ctx, &healthpb.HealthCheckRequest{})+ if err != nil || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING {+ http.Error(w, "not ready", http.StatusServiceUnavailable)+ return+ }+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("ok\n"))+ })++ addr := net.JoinHostPort(harnessHost, strconv.Itoa(readyzPort))+ if err := http.ListenAndServe(addr, mux); err != nil {+ log.Printf("readyz server on %s exited: %v", addr, err)+ }+}diff --git a/cmd/ax/harness_test.go b/cmd/ax/harness_test.goindex 4fbd89c..599f5ab 100644--- a/cmd/ax/harness_test.go+++ b/cmd/ax/harness_test.go@@ -15,9 +15,17 @@package mainimport (+ "fmt"+ "net"+ "net/http""os""path/filepath""testing"+ "time"++ "google.golang.org/grpc"+ "google.golang.org/grpc/health"+ healthpb "google.golang.org/grpc/health/grpc_health_v1")func TestSetHarnessWorkDir(t *testing.T) {@@ -64,3 +72,57 @@ func TestSetHarnessWorkDir(t *testing.T) {}})}++func TestServeReadyz(t *testing.T) {+ // Start a real gRPC server exposing the standard health service; this stands+ // in for the Antigravity Python harness that serveReadyz probes.+ grpcListener, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen for gRPC: %v", err)+ }+ healthServer := health.NewServer()+ grpcServer := grpc.NewServer()+ healthpb.RegisterHealthServer(grpcServer, healthServer)+ go func() { _ = grpcServer.Serve(grpcListener) }()+ t.Cleanup(grpcServer.Stop)+ grpcPort := grpcListener.Addr().(*net.TCPAddr).Port++ // Reserve an ephemeral port for /readyz, then hand it to serveReadyz to bind.+ readyzListener, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen for readyz: %v", err)+ }+ readyzPort := readyzListener.Addr().(*net.TCPAddr).Port+ _ = readyzListener.Close()++ // Start with the harness not serving, then run serveReadyz against it.+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)+ go serveReadyz(readyzPort, grpcPort)+ url := fmt.Sprintf("http://127.0.0.1:%d/readyz", readyzPort)++ // /readyz re-derives readiness from gRPC health on every request: 503 until+ // the harness is SERVING, 200 once it is, and back to 503 if it stops serving.+ waitReadyz(t, url, http.StatusServiceUnavailable)+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)+ waitReadyz(t, url, http.StatusOK)+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)+ waitReadyz(t, url, http.StatusServiceUnavailable)+}++func waitReadyz(t *testing.T, url string, want int) {+ t.Helper()+ deadline := time.Now().Add(5 * time.Second)+ got := 0+ for time.Now().Before(deadline) {+ resp, err := http.Get(url)+ if err == nil {+ got = resp.StatusCode+ _ = resp.Body.Close()+ if got == want {+ return+ }+ }+ time.Sleep(10 * time.Millisecond)+ }+ t.Fatalf("GET %s = %d, want %d", url, got, want)+}diff --git a/go.mod b/go.modindex 308e648..73c022f 100644--- a/go.mod+++ b/go.mod@@ -8,7 +8,7 @@ require (charm.land/huh/v2 v2.0.3charm.land/lipgloss/v2 v2.0.3cloud.google.com/go/compute/metadata v0.9.0- github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27+ github.com/agent-substrate/substrate v0.0.0-20260706222328-3cb7433bd8a8github.com/google/uuid v1.6.0github.com/jackc/pgx/v5 v5.10.0github.com/spf13/cobra v1.10.2diff --git a/go.sum b/go.sumindex 450e5e8..a5bc044 100644--- a/go.sum+++ b/go.sum@@ -10,8 +10,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBcloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=-github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27 h1:REbnncRlgoNQYIHMRm9Gu/3KQReTPQVWRn6A5jZM/GQ=-github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27/go.mod h1:2ri5mTz714LWinayPQG1d4G/kMuzbqvRXlrUrpc7N9Y=+github.com/agent-substrate/substrate v0.0.0-20260706222328-3cb7433bd8a8 h1:tUeBjLs9TJgIWfML2dlZ3UOFG3dacbwrIYkas2ctBgY=+github.com/agent-substrate/substrate v0.0.0-20260706222328-3cb7433bd8a8/go.mod h1:2cvSnnHPwZRAhkrC2LH1bQd1tQBfL3p+zU6pZiHBUkA=github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o=diff --git a/internal/harness/antigravity/antigravity.go b/internal/harness/antigravity/antigravity.goindex 3872644..ed64232 100644--- a/internal/harness/antigravity/antigravity.go+++ b/internal/harness/antigravity/antigravity.go@@ -17,6 +17,7 @@ package antigravityimport ("context""fmt"+ "io""sync""go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"@@ -136,10 +137,13 @@ func (e *antigravityExecution) Run(ctx context.Context, handler harness.Handler)if err != nil {return fmt.Errorf("failed to call gRPC HarnessService.Connect: %w", err)}- if err := stream.Send(start); err != nil {+ // A server that fails before reading the start frame makes Send/CloseSend+ // report io.EOF; the real status is surfaced by DrainStream's Recv below, so+ // only treat non-EOF errors as send failures.+ if err := stream.Send(start); err != nil && err != io.EOF {return fmt.Errorf("failed to send harness start: %w", err)}- if err := stream.CloseSend(); err != nil {+ if err := stream.CloseSend(); err != nil && err != io.EOF {return fmt.Errorf("failed to close stream send direction: %w", err)}diff --git a/internal/harness/substrate/substrate.go b/internal/harness/substrate/substrate.goindex 60a7b52..84d7b45 100644--- a/internal/harness/substrate/substrate.go+++ b/internal/harness/substrate/substrate.go@@ -19,6 +19,7 @@ import ("crypto/tls""errors""fmt"+ "io""log/slog""sync""time"@@ -215,12 +216,15 @@ func (e *substrateExecution) Run(ctx context.Context, handler harness.Handler) e},},}- if err := stream.Send(start); err != nil {+ // A server that fails before reading the start frame makes Send/CloseSend+ // report io.EOF; the real status is surfaced by DrainStream's Recv below, so+ // only treat non-EOF errors as send failures.+ if err := stream.Send(start); err != nil && err != io.EOF {return fmt.Errorf("failed to send harness start: %w", err)}// Close send direction to trigger server processing.- if err := stream.CloseSend(); err != nil {+ if err := stream.CloseSend(); err != nil && err != io.EOF {return fmt.Errorf("failed to close stream send direction: %w", err)}diff --git a/manifests/ax-deployment.yaml b/manifests/ax-deployment.yamlindex 4790d0c..c7e17a0 100644--- a/manifests/ax-deployment.yaml+++ b/manifests/ax-deployment.yaml@@ -67,6 +67,10 @@ spec:value: "/ax/skills"- name: AX_HARNESS_WORKDIRvalue: "/ax"+ readyz:+ httpGet:+ path: /readyz+ port: 8081snapshotsConfig:location: gs://${AX_SNAPSHOTS_BUCKET}/antigravity/---