Agent Executor (AX) is Google's open agentic orchestration runtime
Go
lock,cli: renew Redis lock leases and lengthen suspend/resume timeout (#490)
Siyuan Zhang8d2ccf36f194
diff --git a/cmd/ax/main.go b/cmd/ax/main.goindex d2891ab..dc817d4 100644--- a/cmd/ax/main.go+++ b/cmd/ax/main.go@@ -845,6 +845,14 @@ func manifestFromArgs(args []string) (data []byte, ok bool, err error) {return nil, false, nil}+// lifecycleRPCTimeout bounds suspend/resume RPCs, which wait for Substrate to+// snapshot or restore the task's sandbox. MicroVM-backed sandboxes can take+// well over a minute to snapshot/restore, so this is deliberately generous.+// The server renews its per-task Redis lock for as long as the RPC is in+// flight, so this deadline is not constrained by the lock TTL (see+// lock.RedisLocker).+const lifecycleRPCTimeout = 5 * time.Minute+func runSuspend(serverURL, atespace string, args []string) error {name := ""if len(args) == 1 {@@ -865,7 +873,7 @@ func runSuspend(serverURL, atespace string, args []string) error {}defer conn.Close()- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)+ ctx, cancel := context.WithTimeout(context.Background(), lifecycleRPCTimeout)defer cancel()if _, err := client.SuspendTask(ctx, &v1alpha1.SuspendTaskRequest{Atespace: atespace, Name: name}); err != nil {@@ -896,7 +904,7 @@ func runResume(serverURL, atespace string, args []string) error {}defer conn.Close()- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)+ ctx, cancel := context.WithTimeout(context.Background(), lifecycleRPCTimeout)defer cancel()if _, err := client.ResumeTask(ctx, &v1alpha1.ResumeTaskRequest{Atespace: atespace, Name: name}); err != nil {diff --git a/internal/lock/lock.go b/internal/lock/lock.goindex 0b64549..dd1fb21 100644--- a/internal/lock/lock.go+++ b/internal/lock/lock.go@@ -20,6 +20,7 @@ import ("encoding/hex""errors""fmt"+ "log/slog""sync""time"@@ -117,6 +118,37 @@ func (m *MemoryLocker) Lock(ctx context.Context, kind, atespace, name string) (f// RedisLocker implements distributed locking in Redis using SET NX PX with token validation// and Redis Pub/Sub notification for instant contended lock hand-off.+//+// # Lease renewal+//+// A held lock is kept alive by a background goroutine that extends its TTL+// every TTL/3 (only while the key still holds this holder's token) until the+// unlock function is called. The TTL therefore does not bound how long a lock+// may be held; it only bounds how long a lock can be orphaned after its holder+// crashes or loses connectivity to Redis.+//+// We deliberately renew a short lease instead of using a single long TTL:+//+// - The server does not control how long a critical section runs. Handlers+// hold the lock across Substrate calls (e.g. snapshot/restore of a microVM+// during SuspendTask/ResumeTask) that run under the caller's request+// context, so their duration is set by the client's deadline, not by+// ax-server. A fixed TTL would have to exceed the longest deadline any+// client might use, and would silently allow concurrent holders whenever+// an operation ran past it.+// - The TTL is the recovery time after a crash. A long TTL (minutes) would+// wedge the affected task/workspace/model for that long if ax-server died+// mid-operation; a short renewed lease (30s by default) keeps recovery+// fast.+// - One locker guards every resource kind and operation, so renewal avoids+// having to retune the TTL whenever a slower operation is added.+//+// Limitation: renewal is best-effort and there is no fencing token. If renewals+// keep failing for longer than the TTL (e.g. a prolonged Redis outage), the+// lease can expire while the holder is still working, and the holder is not+// interrupted; the loss is only logged. The alternative considered was a long+// TTL combined with a server-side deadline shorter than it on every locked+// operation, which avoids the goroutine but trades away fast crash recovery.type RedisLocker struct {client *redis.Clientttl time.Duration@@ -124,7 +156,15 @@ type RedisLocker struct {}type RedisLockerOptions struct {- TTL time.Duration+ // TTL is the lease duration of a lock key in Redis (default 30s). Held+ // locks are renewed every TTL/3, so this does not limit how long a lock+ // may be held; it bounds how long a lock survives a crashed holder. See+ // the RedisLocker doc for why a short renewed lease is preferred over a+ // long TTL.+ TTL time.Duration+ // FallbackInterval is how often waiters retry acquisition in case a+ // release notification is missed or a lease expires without a publish+ // (default 1s).FallbackInterval time.Duration}@@ -138,6 +178,15 @@ elseend`)+// renewScript extends the lock TTL only if it is still held by the caller's token.+var renewScript = redis.NewScript(`+if redis.call("get", KEYS[1]) == ARGV[1] then+ return redis.call("pexpire", KEYS[1], ARGV[2])+else+ return 0+end+`)+// ChannelKey returns the Pub/Sub channel used to notify waiters when a lock is released.func ChannelKey(kind, atespace, name string) string {if atespace == "" {@@ -173,10 +222,22 @@ func (r *RedisLocker) Lock(ctx context.Context, kind, atespace, name string) (fu}token := hex.EncodeToString(tokenBytes)+ // makeUnlock is called once the lock is acquired. It starts the lease+ // renewal goroutine (see "Lease renewal" on RedisLocker) and returns an+ // unlock function that stops renewal before releasing the key.makeUnlock := func() func() {+ renewCtx, cancelRenew := context.WithCancel(context.Background())+ done := make(chan struct{})+ go func() {+ defer close(done)+ r.renew(renewCtx, key, token)+ }()+var once sync.Oncereturn func() {once.Do(func() {+ cancelRenew()+ <-donereleaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)defer cancel()_ = releaseAndNotifyScript.Run(releaseCtx, r.client, []string{key, chanKey}, token).Err()@@ -235,3 +296,45 @@ func (r *RedisLocker) Lock(ctx context.Context, kind, atespace, name string) (fu}}}++// renew periodically extends the TTL of the lock identified by key/token until+// ctx is canceled or the lock is found to be no longer held by token.+func (r *RedisLocker) renew(ctx context.Context, key, token string) {+ interval := r.ttl / 3+ if interval <= 0 {+ interval = time.Millisecond+ }+ ticker := time.NewTicker(interval)+ defer ticker.Stop()++ ttlMillis := r.ttl.Milliseconds()+ if ttlMillis <= 0 {+ ttlMillis = 1+ }++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ }++ renewCtx, cancel := context.WithTimeout(ctx, interval)+ res, err := renewScript.Run(renewCtx, r.client, []string{key}, token, ttlMillis).Int()+ cancel()+ if ctx.Err() != nil {+ return+ }+ if err != nil {+ // Transient Redis error: keep trying on the next tick while the+ // remaining TTL still covers us.+ slog.Warn("renewing lock lease", "key", key, "error", err)+ continue+ }+ if res == 0 {+ // The lock expired or was taken over; there is nothing left to renew.+ slog.Error("lock lease lost before release", "key", key)+ return+ }+ }+}diff --git a/internal/lock/lock_test.go b/internal/lock/lock_test.goindex 4128ce7..c887cd1 100644--- a/internal/lock/lock_test.go+++ b/internal/lock/lock_test.go@@ -16,12 +16,16 @@ package lock_testimport ("context"+ "errors"+ "fmt"+ "os""sync""sync/atomic""testing""time""github.com/google/ax/internal/lock"+ "github.com/redis/go-redis/v9")func TestMemoryLocker_SerializesSameResource(t *testing.T) {@@ -94,3 +98,51 @@ func TestMemoryLocker_Timeout(t *testing.T) {t.Fatalf("expected timeout error, got nil")}}++// newTestRedisClient returns a client for the Redis at $REDIS_ADDR, skipping the+// test when it is unset.+func newTestRedisClient(t *testing.T) *redis.Client {+ t.Helper()+ addr := os.Getenv("REDIS_ADDR")+ if addr == "" {+ t.Skip("REDIS_ADDR not set; skipping Redis-backed lock test")+ }+ client := redis.NewClient(&redis.Options{Addr: addr})+ t.Cleanup(func() { _ = client.Close() })+ if err := client.Ping(context.Background()).Err(); err != nil {+ t.Skipf("redis at %s unreachable: %v", addr, err)+ }+ return client+}++// TestRedisLocker_RenewsLeaseWhileHeld verifies that a held lock outlives its+// TTL (because it is renewed) and that waiters acquire it only after release.+func TestRedisLocker_RenewsLeaseWhileHeld(t *testing.T) {+ client := newTestRedisClient(t)+ ttl := 300 * time.Millisecond+ locker := lock.NewRedisLocker(client, lock.RedisLockerOptions{TTL: ttl, FallbackInterval: 20 * time.Millisecond})+ ctx := context.Background()+ name := fmt.Sprintf("renew-%d", time.Now().UnixNano())++ unlock, err := locker.Lock(ctx, "task", "default", name)+ if err != nil {+ t.Fatalf("lock: %v", err)+ }++ // Hold the lock for several TTLs; a contender must not get it meanwhile.+ waitCtx, cancel := context.WithTimeout(ctx, 4*ttl)+ defer cancel()+ if _, err := locker.Lock(waitCtx, "task", "default", name); !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("expected contender deadline while lease was held, got %v", err)+ }++ unlock()++ acquireCtx, cancel2 := context.WithTimeout(ctx, ttl)+ defer cancel2()+ unlock2, err := locker.Lock(acquireCtx, "task", "default", name)+ if err != nil {+ t.Fatalf("lock after release: %v", err)+ }+ unlock2()+}