Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

lock,cli: renew Redis lock leases and lengthen suspend/resume timeout (#490)

Siyuan Zhang8d2ccf36f194
diff --git a/cmd/ax/main.go b/cmd/ax/main.go
index d2891ab..dc817d4 100644
--- a/cmd/ax/main.go
+++ b/cmd/ax/main.go
@@ -845,6 +845,14 @@ func manifestFromArgs(args []string) (data []byte, ok bool, err error) {
return nil, false, nil
}
+// lifecycleRPCTimeout bounds suspend/resume RPCs, which wait for Substrate to
+// snapshot or restore the task's sandbox. MicroVM-backed sandboxes can take
+// well over a minute to snapshot/restore, so this is deliberately generous.
+// The server renews its per-task Redis lock for as long as the RPC is in
+// flight, so this deadline is not constrained by the lock TTL (see
+// lock.RedisLocker).
+const lifecycleRPCTimeout = 5 * time.Minute
+
func runSuspend(serverURL, atespace string, args []string) error {
name := ""
if len(args) == 1 {
@@ -865,7 +873,7 @@ func runSuspend(serverURL, atespace string, args []string) error {
}
defer conn.Close()
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ ctx, cancel := context.WithTimeout(context.Background(), lifecycleRPCTimeout)
defer cancel()
if _, err := client.SuspendTask(ctx, &v1alpha1.SuspendTaskRequest{Atespace: atespace, Name: name}); err != nil {
@@ -896,7 +904,7 @@ func runResume(serverURL, atespace string, args []string) error {
}
defer conn.Close()
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ ctx, cancel := context.WithTimeout(context.Background(), lifecycleRPCTimeout)
defer cancel()
if _, err := client.ResumeTask(ctx, &v1alpha1.ResumeTaskRequest{Atespace: atespace, Name: name}); err != nil {
diff --git a/internal/lock/lock.go b/internal/lock/lock.go
index 0b64549..dd1fb21 100644
--- a/internal/lock/lock.go
+++ b/internal/lock/lock.go
@@ -20,6 +20,7 @@ import (
"encoding/hex"
"errors"
"fmt"
+ "log/slog"
"sync"
"time"
@@ -117,6 +118,37 @@ func (m *MemoryLocker) Lock(ctx context.Context, kind, atespace, name string) (f
// RedisLocker implements distributed locking in Redis using SET NX PX with token validation
// and Redis Pub/Sub notification for instant contended lock hand-off.
+//
+// # Lease renewal
+//
+// A held lock is kept alive by a background goroutine that extends its TTL
+// every TTL/3 (only while the key still holds this holder's token) until the
+// unlock function is called. The TTL therefore does not bound how long a lock
+// may be held; it only bounds how long a lock can be orphaned after its holder
+// crashes or loses connectivity to Redis.
+//
+// We deliberately renew a short lease instead of using a single long TTL:
+//
+// - The server does not control how long a critical section runs. Handlers
+// hold the lock across Substrate calls (e.g. snapshot/restore of a microVM
+// during SuspendTask/ResumeTask) that run under the caller's request
+// context, so their duration is set by the client's deadline, not by
+// ax-server. A fixed TTL would have to exceed the longest deadline any
+// client might use, and would silently allow concurrent holders whenever
+// an operation ran past it.
+// - The TTL is the recovery time after a crash. A long TTL (minutes) would
+// wedge the affected task/workspace/model for that long if ax-server died
+// mid-operation; a short renewed lease (30s by default) keeps recovery
+// fast.
+// - One locker guards every resource kind and operation, so renewal avoids
+// having to retune the TTL whenever a slower operation is added.
+//
+// Limitation: renewal is best-effort and there is no fencing token. If renewals
+// keep failing for longer than the TTL (e.g. a prolonged Redis outage), the
+// lease can expire while the holder is still working, and the holder is not
+// interrupted; the loss is only logged. The alternative considered was a long
+// TTL combined with a server-side deadline shorter than it on every locked
+// operation, which avoids the goroutine but trades away fast crash recovery.
type RedisLocker struct {
client *redis.Client
ttl time.Duration
@@ -124,7 +156,15 @@ type RedisLocker struct {
}
type RedisLockerOptions struct {
- TTL time.Duration
+ // TTL is the lease duration of a lock key in Redis (default 30s). Held
+ // locks are renewed every TTL/3, so this does not limit how long a lock
+ // may be held; it bounds how long a lock survives a crashed holder. See
+ // the RedisLocker doc for why a short renewed lease is preferred over a
+ // long TTL.
+ TTL time.Duration
+ // FallbackInterval is how often waiters retry acquisition in case a
+ // release notification is missed or a lease expires without a publish
+ // (default 1s).
FallbackInterval time.Duration
}
@@ -138,6 +178,15 @@ else
end
`)
+// renewScript extends the lock TTL only if it is still held by the caller's token.
+var renewScript = redis.NewScript(`
+if redis.call("get", KEYS[1]) == ARGV[1] then
+ return redis.call("pexpire", KEYS[1], ARGV[2])
+else
+ return 0
+end
+`)
+
// ChannelKey returns the Pub/Sub channel used to notify waiters when a lock is released.
func ChannelKey(kind, atespace, name string) string {
if atespace == "" {
@@ -173,10 +222,22 @@ func (r *RedisLocker) Lock(ctx context.Context, kind, atespace, name string) (fu
}
token := hex.EncodeToString(tokenBytes)
+ // makeUnlock is called once the lock is acquired. It starts the lease
+ // renewal goroutine (see "Lease renewal" on RedisLocker) and returns an
+ // unlock function that stops renewal before releasing the key.
makeUnlock := func() func() {
+ renewCtx, cancelRenew := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ r.renew(renewCtx, key, token)
+ }()
+
var once sync.Once
return func() {
once.Do(func() {
+ cancelRenew()
+ <-done
releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = releaseAndNotifyScript.Run(releaseCtx, r.client, []string{key, chanKey}, token).Err()
@@ -235,3 +296,45 @@ func (r *RedisLocker) Lock(ctx context.Context, kind, atespace, name string) (fu
}
}
}
+
+// renew periodically extends the TTL of the lock identified by key/token until
+// ctx is canceled or the lock is found to be no longer held by token.
+func (r *RedisLocker) renew(ctx context.Context, key, token string) {
+ interval := r.ttl / 3
+ if interval <= 0 {
+ interval = time.Millisecond
+ }
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+
+ ttlMillis := r.ttl.Milliseconds()
+ if ttlMillis <= 0 {
+ ttlMillis = 1
+ }
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ }
+
+ renewCtx, cancel := context.WithTimeout(ctx, interval)
+ res, err := renewScript.Run(renewCtx, r.client, []string{key}, token, ttlMillis).Int()
+ cancel()
+ if ctx.Err() != nil {
+ return
+ }
+ if err != nil {
+ // Transient Redis error: keep trying on the next tick while the
+ // remaining TTL still covers us.
+ slog.Warn("renewing lock lease", "key", key, "error", err)
+ continue
+ }
+ if res == 0 {
+ // The lock expired or was taken over; there is nothing left to renew.
+ slog.Error("lock lease lost before release", "key", key)
+ return
+ }
+ }
+}
diff --git a/internal/lock/lock_test.go b/internal/lock/lock_test.go
index 4128ce7..c887cd1 100644
--- a/internal/lock/lock_test.go
+++ b/internal/lock/lock_test.go
@@ -16,12 +16,16 @@ package lock_test
import (
"context"
+ "errors"
+ "fmt"
+ "os"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/google/ax/internal/lock"
+ "github.com/redis/go-redis/v9"
)
func TestMemoryLocker_SerializesSameResource(t *testing.T) {
@@ -94,3 +98,51 @@ func TestMemoryLocker_Timeout(t *testing.T) {
t.Fatalf("expected timeout error, got nil")
}
}
+
+// newTestRedisClient returns a client for the Redis at $REDIS_ADDR, skipping the
+// test when it is unset.
+func newTestRedisClient(t *testing.T) *redis.Client {
+ t.Helper()
+ addr := os.Getenv("REDIS_ADDR")
+ if addr == "" {
+ t.Skip("REDIS_ADDR not set; skipping Redis-backed lock test")
+ }
+ client := redis.NewClient(&redis.Options{Addr: addr})
+ t.Cleanup(func() { _ = client.Close() })
+ if err := client.Ping(context.Background()).Err(); err != nil {
+ t.Skipf("redis at %s unreachable: %v", addr, err)
+ }
+ return client
+}
+
+// TestRedisLocker_RenewsLeaseWhileHeld verifies that a held lock outlives its
+// TTL (because it is renewed) and that waiters acquire it only after release.
+func TestRedisLocker_RenewsLeaseWhileHeld(t *testing.T) {
+ client := newTestRedisClient(t)
+ ttl := 300 * time.Millisecond
+ locker := lock.NewRedisLocker(client, lock.RedisLockerOptions{TTL: ttl, FallbackInterval: 20 * time.Millisecond})
+ ctx := context.Background()
+ name := fmt.Sprintf("renew-%d", time.Now().UnixNano())
+
+ unlock, err := locker.Lock(ctx, "task", "default", name)
+ if err != nil {
+ t.Fatalf("lock: %v", err)
+ }
+
+ // Hold the lock for several TTLs; a contender must not get it meanwhile.
+ waitCtx, cancel := context.WithTimeout(ctx, 4*ttl)
+ defer cancel()
+ if _, err := locker.Lock(waitCtx, "task", "default", name); !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("expected contender deadline while lease was held, got %v", err)
+ }
+
+ unlock()
+
+ acquireCtx, cancel2 := context.WithTimeout(ctx, ttl)
+ defer cancel2()
+ unlock2, err := locker.Lock(acquireCtx, "task", "default", name)
+ if err != nil {
+ t.Fatalf("lock after release: %v", err)
+ }
+ unlock2()
+}