Agent Executor (AX) is Google's open agentic orchestration runtime
Go
Enable sidecar PID tracking
JBD34d81b78cff0
diff --git a/cmd/ax/harness.go b/cmd/ax/harness.goindex 5c3541e..7e37395 100644--- a/cmd/ax/harness.go+++ b/cmd/ax/harness.go@@ -119,11 +119,10 @@ func runAntigravityHarness(cmd *cobra.Command) error {"--port", strconv.Itoa(harnessPort),"--state-dir", stateDir,},- Stdout: os.Stdout,- Stderr: os.Stderr,- ReadyFunc: pythonsidecar.TCPReady(addr),- KillOrphans: true,- Address: addr,+ Stdout: os.Stdout,+ Stderr: os.Stderr,+ ReadyFunc: pythonsidecar.TCPReady(addr),+ Address: addr,}sidecar := pythonsidecar.New(cfg)diff --git a/internal/harness/antigravity/antigravity.go b/internal/harness/antigravity/antigravity.goindex 2991a09..ab18ee8 100644--- a/internal/harness/antigravity/antigravity.go+++ b/internal/harness/antigravity/antigravity.go@@ -84,11 +84,10 @@ func New(ctx context.Context, address, stateDir string, autoStart bool) (*Antigrargs = append(args, "--state-dir", stateDir)}cfg := pythonsidecar.Config{- Module: "python.antigravity.harness_server",- Args: args,- ReadyFunc: pythonsidecar.TCPReady(address),- KillOrphans: true,- Address: address,+ Module: "python.antigravity.harness_server",+ Args: args,+ ReadyFunc: pythonsidecar.TCPReady(address),+ Address: address,}sidecar := pythonsidecar.New(cfg)path, err := pythonsidecar.Setup(ctx, pythonsidecar.SetupOptions{diff --git a/internal/pythonsidecar/setup_test.go b/internal/pythonsidecar/setup_test.goindex cfcf3da..fb01ac2 100644--- a/internal/pythonsidecar/setup_test.go+++ b/internal/pythonsidecar/setup_test.go@@ -73,6 +73,7 @@ func TestSetup_EmbeddedFS(t *testing.T) {func TestSidecar_PythonPath(t *testing.T) {tmpDir := t.TempDir()+ t.Setenv("AX_DURABLE_DIR", tmpDir)customPath := filepath.Join(tmpDir, "custom_python_path")if err := os.MkdirAll(customPath, 0755); err != nil {t.Fatalf("failed to create custom path: %v", err)diff --git a/internal/pythonsidecar/sidecar.go b/internal/pythonsidecar/sidecar.goindex 6b25c25..504302b 100644--- a/internal/pythonsidecar/sidecar.go+++ b/internal/pythonsidecar/sidecar.go@@ -18,16 +18,20 @@ package pythonsidecarimport ("context"+ "errors""fmt""io""net""os""os/exec"+ "path/filepath""strconv""strings""sync""syscall""time"++ "github.com/google/ax/internal/config")// Config holds the configuration parameters for the sidecar lifecycle.@@ -43,10 +47,7 @@ type Config struct {// ReadyFunc is an optional function to check if the server is ready to accept requests.// When provided, Start will poll ReadyFunc until it returns nil or the context expires. (Optional)ReadyFunc func(ctx context.Context) error- // KillOrphans when true will attempt to terminate any pre-existing orphan process- // listening on Address before starting the sidecar. (Optional)- KillOrphans bool- // Address is the network host:port address used for orphan process cleanup. (Optional)+ // Address is the network host:port address used for readiness checks / error reporting. (Optional)Address string}@@ -58,7 +59,8 @@ type Sidecar struct {mu sync.Mutexcmd *exec.Cmd- running bool+ pid int+ attached boolstopping boolexitErr errordoneChan chan struct{}@@ -71,12 +73,66 @@ func New(cfg Config) *Sidecar {}}-// Start launches the Python process and monitors its lifecycle in the background.+func resolvePIDFile() (string, error) {+ axDir, err := config.AXAssetsDir()+ if err != nil {+ return "", fmt.Errorf("failed to resolve AX assets dir: %w", err)+ }+ return filepath.Join(axDir, "sidecar.pid"), nil+}++func isAlive(pid int) bool {+ if pid <= 0 {+ return false+ }+ proc, err := os.FindProcess(pid)+ if err != nil {+ return false+ }+ err = proc.Signal(syscall.Signal(0))+ if err == nil || errors.Is(err, syscall.EPERM) {+ return true+ }+ return false+}++func readPID(path string) (int, error) {+ data, err := os.ReadFile(path)+ if err != nil {+ return 0, err+ }+ pidStr := strings.TrimSpace(string(data))+ pid, err := strconv.Atoi(pidStr)+ if err != nil || pid <= 0 {+ return 0, fmt.Errorf("invalid pid in file: %q", pidStr)+ }+ return pid, nil+}++func writePID(path string, pid int) error {+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {+ return err+ }+ return os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0644)+}++func cleanPID(expectedPID int) {+ path, err := resolvePIDFile()+ if err != nil || path == "" {+ return+ }+ pid, err := readPID(path)+ if err == nil && pid == expectedPID {+ _ = os.Remove(path)+ }+}++// Start launches the Python process (or attaches to an existing working one) and monitors its lifecycle.// If ReadyFunc is configured, Start blocks until the server is ready or the context expires.// If the process fails to start or become ready, an error is returned immediately.func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {s.mu.Lock()- if s.running {+ if isAlive(s.pid) {s.mu.Unlock()return fmt.Errorf("sidecar is already running")}@@ -85,16 +141,43 @@ func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {s.mu.Unlock()return fmt.Errorf("Module cannot be empty")}+ s.mu.Unlock()++ pidPath, err := resolvePIDFile()+ if err != nil {+ return err+ }++ // 1. Check if an existing PID is already running and working+ existingPID, readErr := readPID(pidPath)+ if readErr == nil && s.cfg.ReadyFunc != nil && isAlive(existingPID) {+ var readyErr error+ if s.cfg.ReadyFunc != nil {+ readyErr = s.cfg.ReadyFunc(ctx)+ }+ if readyErr == nil {+ // Existing process is alive and working!+ s.mu.Lock()+ s.pid = existingPID+ s.attached = true+ s.stopping = false+ s.exitErr = nil+ s.doneChan = make(chan struct{})+ s.mu.Unlock()++ go s.monitor()+ return nil+ }+ }++ // Existing PID is missing, dead, or not working. Clean up stale PID file.+ if readErr == nil {+ _ = os.Remove(pidPath)+ }if s.cfg.ReadyFunc != nil {if err := s.cfg.ReadyFunc(ctx); err == nil {- if s.cfg.KillOrphans && s.cfg.Address != "" {- _ = killOrphanProcessOnAddr(ctx, s.cfg.Address)- }- if err := s.cfg.ReadyFunc(ctx); err == nil {- s.mu.Unlock()- return fmt.Errorf("cannot start sidecar: endpoint %s is already in use by another process", s.cfg.Address)- }+ return fmt.Errorf("cannot start sidecar: endpoint %s is already in use by another process", s.cfg.Address)}}@@ -131,18 +214,25 @@ func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {cmd.Stderr = s.cfg.Stderr}+ s.mu.Lock()if err := cmd.Start(); err != nil {s.mu.Unlock()return fmt.Errorf("failed to start python process: %w", err)}s.cmd = cmd- s.running = true+ s.pid = cmd.Process.Pid+ s.attached = falses.stopping = falses.exitErr = nils.doneChan = make(chan struct{})s.mu.Unlock()+ if err := writePID(pidPath, s.pid); err != nil {+ _ = s.Stop()+ return fmt.Errorf("failed to write PID file: %w", err)+ }+// Monitor lifecycle asynchronouslygo s.monitor()@@ -157,17 +247,49 @@ func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {return nil}-// monitor waits for the process to exit and records its exit status.+// monitor waits for the process (spawned or attached) to exit and records its exit status.func (s *Sidecar) monitor() {- err := s.cmd.Wait()s.mu.Lock()- s.running = false- if err != nil && !s.stopping {- s.exitErr = fmt.Errorf("python process exited with error: %w", err)- } else {- s.exitErr = nil+ cmd := s.cmd+ pid := s.pid+ attached := s.attached+ s.mu.Unlock()++ var exitErr error+ if attached {+ ticker := time.NewTicker(500 * time.Millisecond)+ defer ticker.Stop()++ for {+ <-ticker.C+ s.mu.Lock()+ stopping := s.stopping+ s.mu.Unlock()+ if stopping {+ break+ }++ if !isAlive(pid) {+ exitErr = fmt.Errorf("attached python process %d exited", pid)+ break+ }+ }+ } else if cmd != nil {+ err := cmd.Wait()+ s.mu.Lock()+ stopping := s.stopping+ s.mu.Unlock()+ if err != nil && !stopping {+ exitErr = fmt.Errorf("python process exited with error: %w", err)+ }}++ s.mu.Lock()+ s.pid = 0+ s.exitErr = exitErrs.mu.Unlock()++ cleanPID(pid)close(s.doneChan)}@@ -175,17 +297,17 @@ func (s *Sidecar) monitor() {func (s *Sidecar) IsRunning() bool {s.mu.Lock()defer s.mu.Unlock()- return s.running+ return isAlive(s.pid)}// Pid returns the process ID of the running sidecar, or 0 if not running.func (s *Sidecar) Pid() int {s.mu.Lock()defer s.mu.Unlock()- if !s.running || s.cmd == nil || s.cmd.Process == nil {+ if !isAlive(s.pid) {return 0}- return s.cmd.Process.Pid+ return s.pid}// WaitUntilReady blocks until ReadyFunc returns nil, the context is canceled, or the process exits prematurely.@@ -201,10 +323,10 @@ func (s *Sidecar) WaitUntilReady(ctx context.Context) error {for {// 1. Check if the process exited prematurelys.mu.Lock()- running := s.running+ pid := s.pidexitErr := s.exitErrs.mu.Unlock()- if !running {+ if !isAlive(pid) {if exitErr != nil {return fmt.Errorf("process exited before becoming ready: %w", exitErr)}@@ -242,17 +364,39 @@ func (s *Sidecar) Wait() error {// Stop gracefully terminates the Python process using SIGTERM, falling back to SIGKILL if necessary.func (s *Sidecar) Stop() error {s.mu.Lock()- if !s.running || s.cmd == nil || s.cmd.Process == nil {+ if !isAlive(s.pid) {s.mu.Unlock()return nil}s.stopping = truedone := s.doneChan+ cmd := s.cmd+ pid := s.pid+ attached := s.attacheds.mu.Unlock()+ defer cleanPID(pid)++ if attached {+ if proc, err := os.FindProcess(pid); err == nil {+ _ = proc.Signal(syscall.SIGTERM)+ }+ if done != nil {+ select {+ case <-done:+ case <-time.After(1 * time.Second):+ }+ }+ return nil+ }++ if cmd == nil || cmd.Process == nil {+ return nil+ }+// 1. Send graceful SIGTERM- if err := s.cmd.Process.Signal(syscall.SIGTERM); err != nil {- _ = s.cmd.Process.Kill()+ if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {+ _ = cmd.Process.Kill()if done != nil {<-done}@@ -265,7 +409,7 @@ func (s *Sidecar) Stop() error {return nilcase <-time.After(3 * time.Second):// Fallback to force kill- _ = s.cmd.Process.Kill()+ _ = cmd.Process.Kill()if done != nil {<-done}@@ -286,58 +430,5 @@ func TCPReady(addr string) func(ctx context.Context) error {}}-// killOrphanProcessOnAddr attempts to terminate pre-existing processes listening on addr.-func killOrphanProcessOnAddr(ctx context.Context, addr string) error {- _, portStr, err := net.SplitHostPort(addr)- if err != nil {- return err- }- port, err := strconv.Atoi(portStr)- if err != nil || port <= 0 {- return fmt.Errorf("invalid port %q", portStr)- }-- out, err := exec.CommandContext(ctx, "lsof", "-ti", fmt.Sprintf("tcp:%d", port)).Output()- if err != nil || len(out) == 0 {- return nil- }-- pidStrs := strings.Fields(string(out))- myPid := os.Getpid()- for _, pStr := range pidStrs {- pid, err := strconv.Atoi(pStr)- if err != nil || pid <= 0 || pid == myPid {- continue- }- proc, err := os.FindProcess(pid)- if err != nil {- continue- }- _ = proc.Signal(syscall.SIGTERM)- }-- deadline := time.Now().Add(1 * time.Second)- for time.Now().Before(deadline) {- var d net.Dialer- conn, err := d.DialContext(ctx, "tcp", addr)- if err != nil {- return nil- }- _ = conn.Close()- time.Sleep(50 * time.Millisecond)- }- for _, pStr := range pidStrs {- pid, err := strconv.Atoi(pStr)- if err != nil || pid <= 0 || pid == myPid {- continue- }- proc, err := os.FindProcess(pid)- if err != nil {- continue- }- _ = proc.Kill()- }- return nil-}diff --git a/internal/pythonsidecar/sidecar_test.go b/internal/pythonsidecar/sidecar_test.goindex 6e469bb..7ff864f 100644--- a/internal/pythonsidecar/sidecar_test.go+++ b/internal/pythonsidecar/sidecar_test.go@@ -54,6 +54,7 @@ func TestSidecar_ConfigValidation(t *testing.T) {func TestSidecar_ModuleExecution(t *testing.T) {tmpDir := t.TempDir()+ t.Setenv("AX_DURABLE_DIR", tmpDir)modulePath := filepath.Join(tmpDir, "test_module.py")moduleContent := `import sys@@ -96,6 +97,7 @@ sys.exit(0)}func TestSidecar_ModuleServerWithTCPReady(t *testing.T) {+ t.Setenv("AX_DURABLE_DIR", t.TempDir())port := getFreePort(t)addr := fmt.Sprintf("127.0.0.1:%d", port)@@ -127,6 +129,7 @@ func TestSidecar_ModuleServerWithTCPReady(t *testing.T) {func TestSidecar_ReadinessFailureOnPrematureExit(t *testing.T) {tmpDir := t.TempDir()+ t.Setenv("AX_DURABLE_DIR", tmpDir)modulePath := filepath.Join(tmpDir, "crash.py")if err := os.WriteFile(modulePath, []byte("import sys; sys.exit(1)\n"), 0644); err != nil {t.Fatalf("failed to write script: %v", err)@@ -158,6 +161,7 @@ func TestSidecar_ReadinessFailureOnPrematureExit(t *testing.T) {}func TestSidecar_EndpointAlreadyInUse(t *testing.T) {+ t.Setenv("AX_DURABLE_DIR", t.TempDir())// Start a dummy TCP listener on a free portl, err := net.Listen("tcp", "127.0.0.1:0")if err != nil {@@ -184,39 +188,90 @@ func TestSidecar_EndpointAlreadyInUse(t *testing.T) {}}-func TestSidecar_KillOrphans(t *testing.T) {+func TestSidecar_PIDFileHandling(t *testing.T) {+ durableDir := t.TempDir()+ t.Setenv("AX_DURABLE_DIR", durableDir)+ pidFile := filepath.Join(durableDir, ".ax", "sidecar.pid")+port := getFreePort(t)addr := fmt.Sprintf("127.0.0.1:%d", port)- // Start a dummy sidecar listening on `port`- dummyCfg := pythonsidecar.Config{+ cfg1 := pythonsidecar.Config{Module: "http.server",Args: []string{strconv.Itoa(port), "--bind", "127.0.0.1"},ReadyFunc: pythonsidecar.TCPReady(addr),}- dummySidecar := pythonsidecar.New(dummyCfg)++ s1 := pythonsidecar.New(cfg1)ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()- if err := dummySidecar.Start(ctx, ""); err != nil {- t.Fatalf("failed to start dummy sidecar: %v", err)+ if err := s1.Start(ctx, ""); err != nil {+ t.Fatalf("s1.Start() failed: %v", err)}- // Now start a new sidecar with KillOrphans: true on the same addr- newCfg := pythonsidecar.Config{- Module: "http.server",- Args: []string{strconv.Itoa(port), "--bind", "127.0.0.1"},- ReadyFunc: pythonsidecar.TCPReady(addr),- KillOrphans: true,- Address: addr,+ // 1. Verify PID file was written in AX config directory+ data, err := os.ReadFile(pidFile)+ if err != nil {+ t.Fatalf("failed to read PID file at %s: %v", pidFile, err)+ }+ filePID, err := strconv.Atoi(strings.TrimSpace(string(data)))+ if err != nil || filePID != s1.Pid() {+ t.Fatalf("PID file content mismatch: got %d, expected %d", filePID, s1.Pid())}- newSidecar := pythonsidecar.New(newCfg)- if err := newSidecar.Start(ctx, ""); err != nil {- t.Fatalf("Start() with KillOrphans failed: %v", err)++ // 2. Verify second sidecar instance adopts the working PID from PID file+ cfg2 := pythonsidecar.Config{+ Module: "http.server",+ Args: []string{strconv.Itoa(port), "--bind", "127.0.0.1"},+ ReadyFunc: pythonsidecar.TCPReady(addr),+ }+ s2 := pythonsidecar.New(cfg2)+ if err := s2.Start(ctx, ""); err != nil {+ t.Fatalf("s2.Start() failed to attach to working PID: %v", err)+ }++ if s2.Pid() != s1.Pid() {+ t.Fatalf("s2 PID %d does not match s1 working PID %d", s2.Pid(), s1.Pid())+ }++ // Stop s1+ if err := s1.Stop(); err != nil {+ t.Fatalf("s1.Stop() failed: %v", err)}- defer newSidecar.Stop()- if !newSidecar.IsRunning() {- t.Fatalf("expected new sidecar to be running")+ // 3. Test dead PID cleanup+ if err := os.MkdirAll(filepath.Dir(pidFile), 0755); err != nil {+ t.Fatalf("failed to create dir for dead PID file: %v", err)+ }+ if err := os.WriteFile(pidFile, []byte("999999\n"), 0644); err != nil {+ t.Fatalf("failed to write dead PID file: %v", err)+ }++ port2 := getFreePort(t)+ addr2 := fmt.Sprintf("127.0.0.1:%d", port2)+ cfg3 := pythonsidecar.Config{+ Module: "http.server",+ Args: []string{strconv.Itoa(port2), "--bind", "127.0.0.1"},+ ReadyFunc: pythonsidecar.TCPReady(addr2),+ }+ s3 := pythonsidecar.New(cfg3)+ if err := s3.Start(ctx, ""); err != nil {+ t.Fatalf("s3.Start() failed when replacing dead PID: %v", err)+ }+ defer s3.Stop()++ if s3.Pid() == 999999 {+ t.Fatalf("s3 should have started a new process, but got dead PID 999999")+ }++ newData, err := os.ReadFile(pidFile)+ if err != nil {+ t.Fatalf("failed to read updated PID file: %v", err)+ }+ newPID, _ := strconv.Atoi(strings.TrimSpace(string(newData)))+ if newPID != s3.Pid() {+ t.Fatalf("updated PID file mismatch: got %d, expected %d", newPID, s3.Pid())}}+