Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

Enable sidecar PID tracking

JBD34d81b78cff0
diff --git a/cmd/ax/harness.go b/cmd/ax/harness.go
index 5c3541e..7e37395 100644
--- a/cmd/ax/harness.go
+++ b/cmd/ax/harness.go
@@ -119,11 +119,10 @@ func runAntigravityHarness(cmd *cobra.Command) error {
"--port", strconv.Itoa(harnessPort),
"--state-dir", stateDir,
},
- Stdout: os.Stdout,
- Stderr: os.Stderr,
- ReadyFunc: pythonsidecar.TCPReady(addr),
- KillOrphans: true,
- Address: addr,
+ Stdout: os.Stdout,
+ Stderr: os.Stderr,
+ ReadyFunc: pythonsidecar.TCPReady(addr),
+ Address: addr,
}
sidecar := pythonsidecar.New(cfg)
diff --git a/internal/harness/antigravity/antigravity.go b/internal/harness/antigravity/antigravity.go
index 2991a09..ab18ee8 100644
--- a/internal/harness/antigravity/antigravity.go
+++ b/internal/harness/antigravity/antigravity.go
@@ -84,11 +84,10 @@ func New(ctx context.Context, address, stateDir string, autoStart bool) (*Antigr
args = append(args, "--state-dir", stateDir)
}
cfg := pythonsidecar.Config{
- Module: "python.antigravity.harness_server",
- Args: args,
- ReadyFunc: pythonsidecar.TCPReady(address),
- KillOrphans: true,
- Address: address,
+ Module: "python.antigravity.harness_server",
+ Args: args,
+ ReadyFunc: pythonsidecar.TCPReady(address),
+ Address: address,
}
sidecar := pythonsidecar.New(cfg)
path, err := pythonsidecar.Setup(ctx, pythonsidecar.SetupOptions{
diff --git a/internal/pythonsidecar/setup_test.go b/internal/pythonsidecar/setup_test.go
index cfcf3da..fb01ac2 100644
--- a/internal/pythonsidecar/setup_test.go
+++ b/internal/pythonsidecar/setup_test.go
@@ -73,6 +73,7 @@ func TestSetup_EmbeddedFS(t *testing.T) {
func TestSidecar_PythonPath(t *testing.T) {
tmpDir := t.TempDir()
+ t.Setenv("AX_DURABLE_DIR", tmpDir)
customPath := filepath.Join(tmpDir, "custom_python_path")
if err := os.MkdirAll(customPath, 0755); err != nil {
t.Fatalf("failed to create custom path: %v", err)
diff --git a/internal/pythonsidecar/sidecar.go b/internal/pythonsidecar/sidecar.go
index 6b25c25..504302b 100644
--- a/internal/pythonsidecar/sidecar.go
+++ b/internal/pythonsidecar/sidecar.go
@@ -18,16 +18,20 @@ package pythonsidecar
import (
"context"
+ "errors"
"fmt"
"io"
"net"
"os"
"os/exec"
+ "path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
+
+ "github.com/google/ax/internal/config"
)
// Config holds the configuration parameters for the sidecar lifecycle.
@@ -43,10 +47,7 @@ type Config struct {
// ReadyFunc is an optional function to check if the server is ready to accept requests.
// When provided, Start will poll ReadyFunc until it returns nil or the context expires. (Optional)
ReadyFunc func(ctx context.Context) error
- // KillOrphans when true will attempt to terminate any pre-existing orphan process
- // listening on Address before starting the sidecar. (Optional)
- KillOrphans bool
- // Address is the network host:port address used for orphan process cleanup. (Optional)
+ // Address is the network host:port address used for readiness checks / error reporting. (Optional)
Address string
}
@@ -58,7 +59,8 @@ type Sidecar struct {
mu sync.Mutex
cmd *exec.Cmd
- running bool
+ pid int
+ attached bool
stopping bool
exitErr error
doneChan chan struct{}
@@ -71,12 +73,66 @@ func New(cfg Config) *Sidecar {
}
}
-// Start launches the Python process and monitors its lifecycle in the background.
+func resolvePIDFile() (string, error) {
+ axDir, err := config.AXAssetsDir()
+ if err != nil {
+ return "", fmt.Errorf("failed to resolve AX assets dir: %w", err)
+ }
+ return filepath.Join(axDir, "sidecar.pid"), nil
+}
+
+func isAlive(pid int) bool {
+ if pid <= 0 {
+ return false
+ }
+ proc, err := os.FindProcess(pid)
+ if err != nil {
+ return false
+ }
+ err = proc.Signal(syscall.Signal(0))
+ if err == nil || errors.Is(err, syscall.EPERM) {
+ return true
+ }
+ return false
+}
+
+func readPID(path string) (int, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return 0, err
+ }
+ pidStr := strings.TrimSpace(string(data))
+ pid, err := strconv.Atoi(pidStr)
+ if err != nil || pid <= 0 {
+ return 0, fmt.Errorf("invalid pid in file: %q", pidStr)
+ }
+ return pid, nil
+}
+
+func writePID(path string, pid int) error {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return err
+ }
+ return os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0644)
+}
+
+func cleanPID(expectedPID int) {
+ path, err := resolvePIDFile()
+ if err != nil || path == "" {
+ return
+ }
+ pid, err := readPID(path)
+ if err == nil && pid == expectedPID {
+ _ = os.Remove(path)
+ }
+}
+
+// Start launches the Python process (or attaches to an existing working one) and monitors its lifecycle.
// If ReadyFunc is configured, Start blocks until the server is ready or the context expires.
// If the process fails to start or become ready, an error is returned immediately.
func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {
s.mu.Lock()
- if s.running {
+ if isAlive(s.pid) {
s.mu.Unlock()
return fmt.Errorf("sidecar is already running")
}
@@ -85,16 +141,43 @@ func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {
s.mu.Unlock()
return fmt.Errorf("Module cannot be empty")
}
+ s.mu.Unlock()
+
+ pidPath, err := resolvePIDFile()
+ if err != nil {
+ return err
+ }
+
+ // 1. Check if an existing PID is already running and working
+ existingPID, readErr := readPID(pidPath)
+ if readErr == nil && s.cfg.ReadyFunc != nil && isAlive(existingPID) {
+ var readyErr error
+ if s.cfg.ReadyFunc != nil {
+ readyErr = s.cfg.ReadyFunc(ctx)
+ }
+ if readyErr == nil {
+ // Existing process is alive and working!
+ s.mu.Lock()
+ s.pid = existingPID
+ s.attached = true
+ s.stopping = false
+ s.exitErr = nil
+ s.doneChan = make(chan struct{})
+ s.mu.Unlock()
+
+ go s.monitor()
+ return nil
+ }
+ }
+
+ // Existing PID is missing, dead, or not working. Clean up stale PID file.
+ if readErr == nil {
+ _ = os.Remove(pidPath)
+ }
if s.cfg.ReadyFunc != nil {
if err := s.cfg.ReadyFunc(ctx); err == nil {
- if s.cfg.KillOrphans && s.cfg.Address != "" {
- _ = killOrphanProcessOnAddr(ctx, s.cfg.Address)
- }
- if err := s.cfg.ReadyFunc(ctx); err == nil {
- s.mu.Unlock()
- return fmt.Errorf("cannot start sidecar: endpoint %s is already in use by another process", s.cfg.Address)
- }
+ return fmt.Errorf("cannot start sidecar: endpoint %s is already in use by another process", s.cfg.Address)
}
}
@@ -131,18 +214,25 @@ func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {
cmd.Stderr = s.cfg.Stderr
}
+ s.mu.Lock()
if err := cmd.Start(); err != nil {
s.mu.Unlock()
return fmt.Errorf("failed to start python process: %w", err)
}
s.cmd = cmd
- s.running = true
+ s.pid = cmd.Process.Pid
+ s.attached = false
s.stopping = false
s.exitErr = nil
s.doneChan = make(chan struct{})
s.mu.Unlock()
+ if err := writePID(pidPath, s.pid); err != nil {
+ _ = s.Stop()
+ return fmt.Errorf("failed to write PID file: %w", err)
+ }
+
// Monitor lifecycle asynchronously
go s.monitor()
@@ -157,17 +247,49 @@ func (s *Sidecar) Start(ctx context.Context, pythonPath string) error {
return nil
}
-// monitor waits for the process to exit and records its exit status.
+// monitor waits for the process (spawned or attached) to exit and records its exit status.
func (s *Sidecar) monitor() {
- err := s.cmd.Wait()
s.mu.Lock()
- s.running = false
- if err != nil && !s.stopping {
- s.exitErr = fmt.Errorf("python process exited with error: %w", err)
- } else {
- s.exitErr = nil
+ cmd := s.cmd
+ pid := s.pid
+ attached := s.attached
+ s.mu.Unlock()
+
+ var exitErr error
+ if attached {
+ ticker := time.NewTicker(500 * time.Millisecond)
+ defer ticker.Stop()
+
+ for {
+ <-ticker.C
+ s.mu.Lock()
+ stopping := s.stopping
+ s.mu.Unlock()
+ if stopping {
+ break
+ }
+
+ if !isAlive(pid) {
+ exitErr = fmt.Errorf("attached python process %d exited", pid)
+ break
+ }
+ }
+ } else if cmd != nil {
+ err := cmd.Wait()
+ s.mu.Lock()
+ stopping := s.stopping
+ s.mu.Unlock()
+ if err != nil && !stopping {
+ exitErr = fmt.Errorf("python process exited with error: %w", err)
+ }
}
+
+ s.mu.Lock()
+ s.pid = 0
+ s.exitErr = exitErr
s.mu.Unlock()
+
+ cleanPID(pid)
close(s.doneChan)
}
@@ -175,17 +297,17 @@ func (s *Sidecar) monitor() {
func (s *Sidecar) IsRunning() bool {
s.mu.Lock()
defer s.mu.Unlock()
- return s.running
+ return isAlive(s.pid)
}
// Pid returns the process ID of the running sidecar, or 0 if not running.
func (s *Sidecar) Pid() int {
s.mu.Lock()
defer s.mu.Unlock()
- if !s.running || s.cmd == nil || s.cmd.Process == nil {
+ if !isAlive(s.pid) {
return 0
}
- return s.cmd.Process.Pid
+ return s.pid
}
// WaitUntilReady blocks until ReadyFunc returns nil, the context is canceled, or the process exits prematurely.
@@ -201,10 +323,10 @@ func (s *Sidecar) WaitUntilReady(ctx context.Context) error {
for {
// 1. Check if the process exited prematurely
s.mu.Lock()
- running := s.running
+ pid := s.pid
exitErr := s.exitErr
s.mu.Unlock()
- if !running {
+ if !isAlive(pid) {
if exitErr != nil {
return fmt.Errorf("process exited before becoming ready: %w", exitErr)
}
@@ -242,17 +364,39 @@ func (s *Sidecar) Wait() error {
// Stop gracefully terminates the Python process using SIGTERM, falling back to SIGKILL if necessary.
func (s *Sidecar) Stop() error {
s.mu.Lock()
- if !s.running || s.cmd == nil || s.cmd.Process == nil {
+ if !isAlive(s.pid) {
s.mu.Unlock()
return nil
}
s.stopping = true
done := s.doneChan
+ cmd := s.cmd
+ pid := s.pid
+ attached := s.attached
s.mu.Unlock()
+ defer cleanPID(pid)
+
+ if attached {
+ if proc, err := os.FindProcess(pid); err == nil {
+ _ = proc.Signal(syscall.SIGTERM)
+ }
+ if done != nil {
+ select {
+ case <-done:
+ case <-time.After(1 * time.Second):
+ }
+ }
+ return nil
+ }
+
+ if cmd == nil || cmd.Process == nil {
+ return nil
+ }
+
// 1. Send graceful SIGTERM
- if err := s.cmd.Process.Signal(syscall.SIGTERM); err != nil {
- _ = s.cmd.Process.Kill()
+ if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+ _ = cmd.Process.Kill()
if done != nil {
<-done
}
@@ -265,7 +409,7 @@ func (s *Sidecar) Stop() error {
return nil
case <-time.After(3 * time.Second):
// Fallback to force kill
- _ = s.cmd.Process.Kill()
+ _ = cmd.Process.Kill()
if done != nil {
<-done
}
@@ -286,58 +430,5 @@ func TCPReady(addr string) func(ctx context.Context) error {
}
}
-// killOrphanProcessOnAddr attempts to terminate pre-existing processes listening on addr.
-func killOrphanProcessOnAddr(ctx context.Context, addr string) error {
- _, portStr, err := net.SplitHostPort(addr)
- if err != nil {
- return err
- }
- port, err := strconv.Atoi(portStr)
- if err != nil || port <= 0 {
- return fmt.Errorf("invalid port %q", portStr)
- }
-
- out, err := exec.CommandContext(ctx, "lsof", "-ti", fmt.Sprintf("tcp:%d", port)).Output()
- if err != nil || len(out) == 0 {
- return nil
- }
-
- pidStrs := strings.Fields(string(out))
- myPid := os.Getpid()
- for _, pStr := range pidStrs {
- pid, err := strconv.Atoi(pStr)
- if err != nil || pid <= 0 || pid == myPid {
- continue
- }
- proc, err := os.FindProcess(pid)
- if err != nil {
- continue
- }
- _ = proc.Signal(syscall.SIGTERM)
- }
-
- deadline := time.Now().Add(1 * time.Second)
- for time.Now().Before(deadline) {
- var d net.Dialer
- conn, err := d.DialContext(ctx, "tcp", addr)
- if err != nil {
- return nil
- }
- _ = conn.Close()
- time.Sleep(50 * time.Millisecond)
- }
- for _, pStr := range pidStrs {
- pid, err := strconv.Atoi(pStr)
- if err != nil || pid <= 0 || pid == myPid {
- continue
- }
- proc, err := os.FindProcess(pid)
- if err != nil {
- continue
- }
- _ = proc.Kill()
- }
- return nil
-}
diff --git a/internal/pythonsidecar/sidecar_test.go b/internal/pythonsidecar/sidecar_test.go
index 6e469bb..7ff864f 100644
--- a/internal/pythonsidecar/sidecar_test.go
+++ b/internal/pythonsidecar/sidecar_test.go
@@ -54,6 +54,7 @@ func TestSidecar_ConfigValidation(t *testing.T) {
func TestSidecar_ModuleExecution(t *testing.T) {
tmpDir := t.TempDir()
+ t.Setenv("AX_DURABLE_DIR", tmpDir)
modulePath := filepath.Join(tmpDir, "test_module.py")
moduleContent := `
import sys
@@ -96,6 +97,7 @@ sys.exit(0)
}
func TestSidecar_ModuleServerWithTCPReady(t *testing.T) {
+ t.Setenv("AX_DURABLE_DIR", t.TempDir())
port := getFreePort(t)
addr := fmt.Sprintf("127.0.0.1:%d", port)
@@ -127,6 +129,7 @@ func TestSidecar_ModuleServerWithTCPReady(t *testing.T) {
func TestSidecar_ReadinessFailureOnPrematureExit(t *testing.T) {
tmpDir := t.TempDir()
+ t.Setenv("AX_DURABLE_DIR", tmpDir)
modulePath := filepath.Join(tmpDir, "crash.py")
if err := os.WriteFile(modulePath, []byte("import sys; sys.exit(1)\n"), 0644); err != nil {
t.Fatalf("failed to write script: %v", err)
@@ -158,6 +161,7 @@ func TestSidecar_ReadinessFailureOnPrematureExit(t *testing.T) {
}
func TestSidecar_EndpointAlreadyInUse(t *testing.T) {
+ t.Setenv("AX_DURABLE_DIR", t.TempDir())
// Start a dummy TCP listener on a free port
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
@@ -184,39 +188,90 @@ func TestSidecar_EndpointAlreadyInUse(t *testing.T) {
}
}
-func TestSidecar_KillOrphans(t *testing.T) {
+func TestSidecar_PIDFileHandling(t *testing.T) {
+ durableDir := t.TempDir()
+ t.Setenv("AX_DURABLE_DIR", durableDir)
+ pidFile := filepath.Join(durableDir, ".ax", "sidecar.pid")
+
port := getFreePort(t)
addr := fmt.Sprintf("127.0.0.1:%d", port)
- // Start a dummy sidecar listening on `port`
- dummyCfg := pythonsidecar.Config{
+ cfg1 := pythonsidecar.Config{
Module: "http.server",
Args: []string{strconv.Itoa(port), "--bind", "127.0.0.1"},
ReadyFunc: pythonsidecar.TCPReady(addr),
}
- dummySidecar := pythonsidecar.New(dummyCfg)
+
+ s1 := pythonsidecar.New(cfg1)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
- if err := dummySidecar.Start(ctx, ""); err != nil {
- t.Fatalf("failed to start dummy sidecar: %v", err)
+ if err := s1.Start(ctx, ""); err != nil {
+ t.Fatalf("s1.Start() failed: %v", err)
}
- // Now start a new sidecar with KillOrphans: true on the same addr
- newCfg := pythonsidecar.Config{
- Module: "http.server",
- Args: []string{strconv.Itoa(port), "--bind", "127.0.0.1"},
- ReadyFunc: pythonsidecar.TCPReady(addr),
- KillOrphans: true,
- Address: addr,
+ // 1. Verify PID file was written in AX config directory
+ data, err := os.ReadFile(pidFile)
+ if err != nil {
+ t.Fatalf("failed to read PID file at %s: %v", pidFile, err)
+ }
+ filePID, err := strconv.Atoi(strings.TrimSpace(string(data)))
+ if err != nil || filePID != s1.Pid() {
+ t.Fatalf("PID file content mismatch: got %d, expected %d", filePID, s1.Pid())
}
- newSidecar := pythonsidecar.New(newCfg)
- if err := newSidecar.Start(ctx, ""); err != nil {
- t.Fatalf("Start() with KillOrphans failed: %v", err)
+
+ // 2. Verify second sidecar instance adopts the working PID from PID file
+ cfg2 := pythonsidecar.Config{
+ Module: "http.server",
+ Args: []string{strconv.Itoa(port), "--bind", "127.0.0.1"},
+ ReadyFunc: pythonsidecar.TCPReady(addr),
+ }
+ s2 := pythonsidecar.New(cfg2)
+ if err := s2.Start(ctx, ""); err != nil {
+ t.Fatalf("s2.Start() failed to attach to working PID: %v", err)
+ }
+
+ if s2.Pid() != s1.Pid() {
+ t.Fatalf("s2 PID %d does not match s1 working PID %d", s2.Pid(), s1.Pid())
+ }
+
+ // Stop s1
+ if err := s1.Stop(); err != nil {
+ t.Fatalf("s1.Stop() failed: %v", err)
}
- defer newSidecar.Stop()
- if !newSidecar.IsRunning() {
- t.Fatalf("expected new sidecar to be running")
+ // 3. Test dead PID cleanup
+ if err := os.MkdirAll(filepath.Dir(pidFile), 0755); err != nil {
+ t.Fatalf("failed to create dir for dead PID file: %v", err)
+ }
+ if err := os.WriteFile(pidFile, []byte("999999\n"), 0644); err != nil {
+ t.Fatalf("failed to write dead PID file: %v", err)
+ }
+
+ port2 := getFreePort(t)
+ addr2 := fmt.Sprintf("127.0.0.1:%d", port2)
+ cfg3 := pythonsidecar.Config{
+ Module: "http.server",
+ Args: []string{strconv.Itoa(port2), "--bind", "127.0.0.1"},
+ ReadyFunc: pythonsidecar.TCPReady(addr2),
+ }
+ s3 := pythonsidecar.New(cfg3)
+ if err := s3.Start(ctx, ""); err != nil {
+ t.Fatalf("s3.Start() failed when replacing dead PID: %v", err)
+ }
+ defer s3.Stop()
+
+ if s3.Pid() == 999999 {
+ t.Fatalf("s3 should have started a new process, but got dead PID 999999")
+ }
+
+ newData, err := os.ReadFile(pidFile)
+ if err != nil {
+ t.Fatalf("failed to read updated PID file: %v", err)
+ }
+ newPID, _ := strconv.Atoi(strings.TrimSpace(string(newData)))
+ if newPID != s3.Pid() {
+ t.Fatalf("updated PID file mismatch: got %d, expected %d", newPID, s3.Pid())
}
}
+