Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

Serve /readyz from ax harness server. (#254)

Junjie Wang948ca05b5233
diff --git a/cmd/ax/harness.go b/cmd/ax/harness.go
index cf6470a..a046c77 100644
--- a/cmd/ax/harness.go
+++ b/cmd/ax/harness.go
@@ -18,15 +18,22 @@
package main
import (
+ "context"
"fmt"
"log"
+ "net"
+ "net/http"
"os"
"os/exec"
"os/signal"
"strconv"
"syscall"
+ "time"
"github.com/spf13/cobra"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ healthpb "google.golang.org/grpc/health/grpc_health_v1"
)
var (
@@ -34,6 +41,9 @@ var (
harnessHost string
)
+// harnessReadyzPort is the port for the HTTP /readyz readiness endpoint.
+const harnessReadyzPort = 8081
+
var harnessCmd = &cobra.Command{
Use: "harness",
Short: "Run the harness gRPC server",
@@ -84,6 +94,10 @@ func runHarness(cmd *cobra.Command, args []string) error {
}
log.Printf("forked antigravity harness server (pid %d) on %s:%d", py.Process.Pid, harnessHost, harnessPort)
+ // Serve the /readyz endpoint that substrate's readiness probe polls (during
+ // golden snapshotting and per-actor Run/Restore).
+ go serveReadyz(harnessReadyzPort, harnessPort)
+
// Forward termination signals to the child so substrate can stop the actor.
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
@@ -98,3 +112,36 @@ func runHarness(cmd *cobra.Command, args []string) error {
}
return nil
}
+
+// serveReadyz serves the HTTP /readyz endpoint on readyzPort that substrate's
+// readiness probe polls (during golden snapshotting and per-actor Run/Restore).
+// Each request forwards to the forked harness's gRPC health Check.
+func serveReadyz(readyzPort, grpcPort int) {
+ conn, err := grpc.NewClient(
+ net.JoinHostPort("127.0.0.1", strconv.Itoa(grpcPort)),
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ )
+ if err != nil {
+ log.Printf("readyz: failed to create gRPC health client: %v", err)
+ return
+ }
+ healthClient := healthpb.NewHealthClient(conn)
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
+ ctx, cancel := context.WithTimeout(r.Context(), time.Second)
+ defer cancel()
+ resp, err := healthClient.Check(ctx, &healthpb.HealthCheckRequest{})
+ if err != nil || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING {
+ http.Error(w, "not ready", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok\n"))
+ })
+
+ addr := net.JoinHostPort(harnessHost, strconv.Itoa(readyzPort))
+ if err := http.ListenAndServe(addr, mux); err != nil {
+ log.Printf("readyz server on %s exited: %v", addr, err)
+ }
+}
diff --git a/cmd/ax/harness_test.go b/cmd/ax/harness_test.go
index 4fbd89c..599f5ab 100644
--- a/cmd/ax/harness_test.go
+++ b/cmd/ax/harness_test.go
@@ -15,9 +15,17 @@
package main
import (
+ "fmt"
+ "net"
+ "net/http"
"os"
"path/filepath"
"testing"
+ "time"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/health"
+ healthpb "google.golang.org/grpc/health/grpc_health_v1"
)
func TestSetHarnessWorkDir(t *testing.T) {
@@ -64,3 +72,57 @@ func TestSetHarnessWorkDir(t *testing.T) {
}
})
}
+
+func TestServeReadyz(t *testing.T) {
+ // Start a real gRPC server exposing the standard health service; this stands
+ // in for the Antigravity Python harness that serveReadyz probes.
+ grpcListener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen for gRPC: %v", err)
+ }
+ healthServer := health.NewServer()
+ grpcServer := grpc.NewServer()
+ healthpb.RegisterHealthServer(grpcServer, healthServer)
+ go func() { _ = grpcServer.Serve(grpcListener) }()
+ t.Cleanup(grpcServer.Stop)
+ grpcPort := grpcListener.Addr().(*net.TCPAddr).Port
+
+ // Reserve an ephemeral port for /readyz, then hand it to serveReadyz to bind.
+ readyzListener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen for readyz: %v", err)
+ }
+ readyzPort := readyzListener.Addr().(*net.TCPAddr).Port
+ _ = readyzListener.Close()
+
+ // Start with the harness not serving, then run serveReadyz against it.
+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)
+ go serveReadyz(readyzPort, grpcPort)
+ url := fmt.Sprintf("http://127.0.0.1:%d/readyz", readyzPort)
+
+ // /readyz re-derives readiness from gRPC health on every request: 503 until
+ // the harness is SERVING, 200 once it is, and back to 503 if it stops serving.
+ waitReadyz(t, url, http.StatusServiceUnavailable)
+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)
+ waitReadyz(t, url, http.StatusOK)
+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)
+ waitReadyz(t, url, http.StatusServiceUnavailable)
+}
+
+func waitReadyz(t *testing.T, url string, want int) {
+ t.Helper()
+ deadline := time.Now().Add(5 * time.Second)
+ got := 0
+ for time.Now().Before(deadline) {
+ resp, err := http.Get(url)
+ if err == nil {
+ got = resp.StatusCode
+ _ = resp.Body.Close()
+ if got == want {
+ return
+ }
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ t.Fatalf("GET %s = %d, want %d", url, got, want)
+}
diff --git a/go.mod b/go.mod
index 308e648..73c022f 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
charm.land/huh/v2 v2.0.3
charm.land/lipgloss/v2 v2.0.3
cloud.google.com/go/compute/metadata v0.9.0
- github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27
+ github.com/agent-substrate/substrate v0.0.0-20260706222328-3cb7433bd8a8
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.10.0
github.com/spf13/cobra v1.10.2
diff --git a/go.sum b/go.sum
index 450e5e8..a5bc044 100644
--- a/go.sum
+++ b/go.sum
@@ -10,8 +10,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
-github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27 h1:REbnncRlgoNQYIHMRm9Gu/3KQReTPQVWRn6A5jZM/GQ=
-github.com/agent-substrate/substrate v0.0.0-20260629215754-fe48750deb27/go.mod h1:2ri5mTz714LWinayPQG1d4G/kMuzbqvRXlrUrpc7N9Y=
+github.com/agent-substrate/substrate v0.0.0-20260706222328-3cb7433bd8a8 h1:tUeBjLs9TJgIWfML2dlZ3UOFG3dacbwrIYkas2ctBgY=
+github.com/agent-substrate/substrate v0.0.0-20260706222328-3cb7433bd8a8/go.mod h1:2cvSnnHPwZRAhkrC2LH1bQd1tQBfL3p+zU6pZiHBUkA=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o=
diff --git a/internal/harness/antigravity/antigravity.go b/internal/harness/antigravity/antigravity.go
index 3872644..ed64232 100644
--- a/internal/harness/antigravity/antigravity.go
+++ b/internal/harness/antigravity/antigravity.go
@@ -17,6 +17,7 @@ package antigravity
import (
"context"
"fmt"
+ "io"
"sync"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
@@ -136,10 +137,13 @@ func (e *antigravityExecution) Run(ctx context.Context, handler harness.Handler)
if err != nil {
return fmt.Errorf("failed to call gRPC HarnessService.Connect: %w", err)
}
- if err := stream.Send(start); err != nil {
+ // A server that fails before reading the start frame makes Send/CloseSend
+ // report io.EOF; the real status is surfaced by DrainStream's Recv below, so
+ // only treat non-EOF errors as send failures.
+ if err := stream.Send(start); err != nil && err != io.EOF {
return fmt.Errorf("failed to send harness start: %w", err)
}
- if err := stream.CloseSend(); err != nil {
+ if err := stream.CloseSend(); err != nil && err != io.EOF {
return fmt.Errorf("failed to close stream send direction: %w", err)
}
diff --git a/internal/harness/substrate/substrate.go b/internal/harness/substrate/substrate.go
index 60a7b52..84d7b45 100644
--- a/internal/harness/substrate/substrate.go
+++ b/internal/harness/substrate/substrate.go
@@ -19,6 +19,7 @@ import (
"crypto/tls"
"errors"
"fmt"
+ "io"
"log/slog"
"sync"
"time"
@@ -215,12 +216,15 @@ func (e *substrateExecution) Run(ctx context.Context, handler harness.Handler) e
},
},
}
- if err := stream.Send(start); err != nil {
+ // A server that fails before reading the start frame makes Send/CloseSend
+ // report io.EOF; the real status is surfaced by DrainStream's Recv below, so
+ // only treat non-EOF errors as send failures.
+ if err := stream.Send(start); err != nil && err != io.EOF {
return fmt.Errorf("failed to send harness start: %w", err)
}
// Close send direction to trigger server processing.
- if err := stream.CloseSend(); err != nil {
+ if err := stream.CloseSend(); err != nil && err != io.EOF {
return fmt.Errorf("failed to close stream send direction: %w", err)
}
diff --git a/manifests/ax-deployment.yaml b/manifests/ax-deployment.yaml
index 4790d0c..c7e17a0 100644
--- a/manifests/ax-deployment.yaml
+++ b/manifests/ax-deployment.yaml
@@ -67,6 +67,10 @@ spec:
value: "/ax/skills"
- name: AX_HARNESS_WORKDIR
value: "/ax"
+ readyz:
+ httpGet:
+ path: /readyz
+ port: 8081
snapshotsConfig:
location: gs://${AX_SNAPSHOTS_BUCKET}/antigravity/
---