Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(agent): preserve Stagehand client leases (#20451)

Sbaaoui Idrisse6bc61d713d6
diff --git a/internal/agent/component/stagehand_runtime.go b/internal/agent/component/stagehand_runtime.go
index 7214f1f355..4e6a69852e 100644
--- a/internal/agent/component/stagehand_runtime.go
+++ b/internal/agent/component/stagehand_runtime.go
@@ -46,10 +46,8 @@
//
// # Concurrency
//
-// The cache itself is a sync.Map; reads are lock-free, writes use
-// `LoadOrStore` to dedupe concurrent builds (the losing goroutine
-// closes its orphan subprocess). The sweeper goroutine runs in the
-// background; `Close()` drains it and shuts down every entry.
+// The cache is a sync.Map; a runtime mutex serializes acquisitions and
+// eviction so an entry cannot close while a caller is acquiring a lease.
//
// # Dependency
//
@@ -66,6 +64,7 @@ import (
"fmt"
"math"
"ragflow/internal/common"
+ "sort"
"strconv"
"strings"
"sync"
@@ -212,12 +211,13 @@ func newStagehandRuntimeFromEnv() *stagehandRuntime {
// stagehandClientEntry is one cached stagehand.Client. lastUsedAt
// is touched on every cache hit; the sweeper uses it to decide
-// eviction. closeOnce makes Close idempotent (called both by the
-// sweeper on TTL eviction and by runtime.Close on shutdown).
+// eviction. A retired entry closes once its last operation releases it.
type stagehandClientEntry struct {
client stagehand.Client
lastUsedAt atomic.Int64 // unix nano
closeOnce sync.Once
+ users int // guarded by stagehandRuntime.mu
+ retired bool
}
// Close shuts down the client (kills the stagehand-server
@@ -234,6 +234,9 @@ func (e *stagehandClientEntry) Close() {
// multi-tenant model rationale.
type stagehandRuntime struct {
cache sync.Map // map[string]*stagehandClientEntry
+ mu sync.Mutex
+ closed bool
+ stopOnce sync.Once
ttl time.Duration
cap int // 0 = unlimited
sweepStop chan struct{}
@@ -286,37 +289,57 @@ func (r *stagehandRuntime) startSweeper(interval time.Duration) {
}()
}
-// evictExpired removes entries whose lastUsedAt is older than ttl.
-// Per-entry Close runs after the LoadAndDelete so concurrent
-// readers either see the entry (and continue) or see nothing
-// (and build a fresh one). The losing side of any LoadAndDelete
-// race sees no entry and skips Close.
+// evictExpired retires entries whose lastUsedAt is older than ttl.
+// Active operations hold a lease until their session cleanup finishes.
func (r *stagehandRuntime) evictExpired() {
cutoff := time.Now().Add(-r.ttl).UnixNano()
+ removed := r.retireExpired(cutoff)
+ for _, e := range removed {
+ e.Close()
+ }
+}
+
+func (r *stagehandRuntime) retireExpired(cutoff int64) []*stagehandClientEntry {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ var removed []*stagehandClientEntry
+ var keys []string
r.cache.Range(func(k, v any) bool {
+ keys = append(keys, k.(string))
+ return true
+ })
+ sort.Strings(keys)
+ for _, k := range keys {
+ v, ok := r.cache.Load(k)
+ if !ok {
+ continue
+ }
e := v.(*stagehandClientEntry)
if e.lastUsedAt.Load() >= cutoff {
- return true // still fresh
+ continue // still fresh
}
if cur, loaded := r.cache.LoadAndDelete(k); loaded {
- cur.(*stagehandClientEntry).Close()
+ e = cur.(*stagehandClientEntry)
+ e.retired = true
+ if e.users == 0 {
+ removed = append(removed, e)
+ }
}
- return true
- })
+ }
+ return removed
}
// enforceLRUCap evicts the least-recently-used entry when the cache
-// exceeds `cap`. Called by clientFor after a successful new-entry
-// insert. The Range + LoadAndDelete sequence is not strictly atomic
-// (concurrent inserts may race), but we only need to converge to
-// "size ≤ cap + a small overshoot" — not exact cardinality.
+// exceeds `cap`. Called by leaseClient after a successful insert.
func (r *stagehandRuntime) enforceLRUCap() {
if r.cap <= 0 {
return
}
+ r.mu.Lock()
count := 0
r.cache.Range(func(_, _ any) bool { count++; return true })
if count <= r.cap {
+ r.mu.Unlock()
return
}
var (
@@ -331,10 +354,20 @@ func (r *stagehandRuntime) enforceLRUCap() {
return true
})
if oldestKey == "" {
+ r.mu.Unlock()
return
}
+ var closeEntry *stagehandClientEntry
if cur, loaded := r.cache.LoadAndDelete(oldestKey); loaded {
- cur.(*stagehandClientEntry).Close()
+ e := cur.(*stagehandClientEntry)
+ e.retired = true
+ if e.users == 0 {
+ closeEntry = e
+ }
+ }
+ r.mu.Unlock()
+ if closeEntry != nil {
+ closeEntry.Close()
}
}
@@ -362,10 +395,11 @@ func (r *stagehandRuntime) RunTask(ctx context.Context, req RunTaskRequest) (str
return "", errors.New("stagehand runtime: APIKey is required (stagehand local mode requires MODEL_API_KEY)")
}
- client, err := r.clientFor(req)
+ client, release, err := r.leaseClient(req)
if err != nil {
return "", fmt.Errorf("stagehand runtime: client: %w", err)
}
+ defer release()
headless := true
if req.Headless != nil {
@@ -524,7 +558,7 @@ func (r *stagehandRuntime) RunExtract(ctx context.Context, req RunExtractRequest
return "", errors.New("stagehand runtime: RunExtract: APIKey is required (stagehand local mode requires MODEL_API_KEY)")
}
- client, err := r.clientFor(RunTaskRequest{
+ client, release, err := r.leaseClient(RunTaskRequest{
BaseURL: req.BaseURL,
APIKey: req.APIKey,
ModelName: req.ModelName,
@@ -532,6 +566,7 @@ func (r *stagehandRuntime) RunExtract(ctx context.Context, req RunExtractRequest
if err != nil {
return "", fmt.Errorf("stagehand runtime: RunExtract: client: %w", err)
}
+ defer release()
headless := true
if req.Headless != nil {
@@ -617,60 +652,66 @@ func stagehandExtractModelConfig(req RunExtractRequest) stagehand.SessionExtract
// Safe to call multiple times.
func (r *stagehandRuntime) Close() error {
if r.sweepStop != nil {
- select {
- case <-r.sweepStop:
- // already closed
- default:
- close(r.sweepStop)
- }
+ r.stopOnce.Do(func() { close(r.sweepStop) })
if r.sweepDone != nil {
<-r.sweepDone
}
}
+ r.mu.Lock()
+ r.closed = true
+ var removed []*stagehandClientEntry
r.cache.Range(func(k, v any) bool {
- v.(*stagehandClientEntry).Close()
+ e := v.(*stagehandClientEntry)
r.cache.Delete(k)
+ e.retired = true
+ if e.users == 0 {
+ removed = append(removed, e)
+ }
return true
})
+ r.mu.Unlock()
+ for _, e := range removed {
+ e.Close()
+ }
return nil
}
-// clientFor returns the cached stagehand.Client for req, building
-// + caching a new one on miss. Concurrent calls for the same key
-// deduplicate via sync.Map.LoadOrStore (the loser closes its
-// orphan subprocess).
-func (r *stagehandRuntime) clientFor(req RunTaskRequest) (stagehand.Client, error) {
+func (r *stagehandRuntime) leaseClient(req RunTaskRequest) (stagehand.Client, func(), error) {
if req.APIKey == "" {
- return stagehand.Client{}, errors.New("stagehand runtime: APIKey is required")
+ return stagehand.Client{}, nil, errors.New("stagehand runtime: APIKey is required")
}
key := req.BaseURL + "|" + req.APIKey + "|" + req.ModelName
-
- // Fast path: read-only.
- if v, ok := r.cache.Load(key); ok {
- e := v.(*stagehandClientEntry)
- e.lastUsedAt.Store(time.Now().UnixNano())
- return e.client, nil
+ r.mu.Lock()
+ if r.closed {
+ r.mu.Unlock()
+ return stagehand.Client{}, nil, errors.New("stagehand runtime: closed")
}
-
- // Slow path: build a fresh client, then atomically publish.
- // LoadOrStore dedupes concurrent builds for the same key.
- entry := &stagehandClientEntry{
- client: stagehand.NewClient(
+ var e *stagehandClientEntry
+ inserted := false
+ if v, ok := r.cache.Load(key); ok {
+ e = v.(*stagehandClientEntry)
+ } else {
+ e = &stagehandClientEntry{client: stagehand.NewClient(
option.WithServer("local"),
option.WithModelAPIKey(req.APIKey),
- ),
- }
- entry.lastUsedAt.Store(time.Now().UnixNano())
-
- actual, loaded := r.cache.LoadOrStore(key, entry)
- if loaded {
- // Another goroutine won the race. Close our orphan and
- // adopt theirs. Their entry already has a fresh lastUsedAt.
- entry.Close()
- return actual.(*stagehandClientEntry).client, nil
- }
-
- // We won the publish. Possibly evict the LRU oldest.
- r.enforceLRUCap()
- return entry.client, nil
+ )}
+ r.cache.Store(key, e)
+ inserted = true
+ }
+ e.lastUsedAt.Store(time.Now().UnixNano())
+ e.users++
+ r.mu.Unlock()
+
+ if inserted {
+ r.enforceLRUCap()
+ }
+ return e.client, func() {
+ r.mu.Lock()
+ e.users--
+ closeEntry := e.users == 0 && e.retired
+ r.mu.Unlock()
+ if closeEntry {
+ e.Close()
+ }
+ }, nil
}
diff --git a/internal/agent/component/stagehand_runtime_test.go b/internal/agent/component/stagehand_runtime_test.go
index 056455c485..2178938714 100644
--- a/internal/agent/component/stagehand_runtime_test.go
+++ b/internal/agent/component/stagehand_runtime_test.go
@@ -19,11 +19,17 @@ package component
import (
"context"
"errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
+
+ stagehand "github.com/browserbase/stagehand-go/v3"
+ "github.com/browserbase/stagehand-go/v3/option"
)
// cacheKey is the key formula used by clientFor. Exposed so tests
@@ -39,6 +45,15 @@ func cacheSize(r *stagehandRuntime) int {
return n
}
+// clientFor populates the cache without retaining an operation lease.
+func (r *stagehandRuntime) clientFor(req RunTaskRequest) (stagehand.Client, error) {
+ client, release, err := r.leaseClient(req)
+ if err == nil {
+ release()
+ }
+ return client, err
+}
+
// ---------------------------------------------------------------------------
// Validation (unchanged)
// ---------------------------------------------------------------------------
@@ -470,6 +485,148 @@ func TestStagehandRuntime_Close_TTLZero_NoSweeper(t *testing.T) {
}
}
+type countedStagehandOption struct {
+ option.RequestOption
+ closes atomic.Int32
+}
+
+func (o *countedStagehandOption) Close() error {
+ o.closes.Add(1)
+ return nil
+}
+
+func TestStagehandRuntime_EvictionWaitsForSessionEnd(t *testing.T) {
+ for _, operation := range []string{"task", "extract"} {
+ for _, eviction := range []string{"ttl", "lru", "close"} {
+ t.Run(operation+"/"+eviction, func(t *testing.T) {
+ endStarted := make(chan struct{})
+ finishEnd := make(chan struct{})
+ releaseEnd := sync.OnceFunc(func() { close(finishEnd) })
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(req.URL.Path, "/start"):
+ _, _ = fmt.Fprint(w, `{"success":true,"data":{"sessionId":"s"}}`)
+ case strings.HasSuffix(req.URL.Path, "/end"):
+ close(endStarted)
+ <-finishEnd
+ _, _ = fmt.Fprint(w, `{"success":true}`)
+ case strings.HasSuffix(req.URL.Path, "/agentExecute"):
+ _, _ = fmt.Fprint(w, `{"success":true,"data":{"result":{"success":true,"message":"ok"}}}`)
+ case strings.HasSuffix(req.URL.Path, "/extract"):
+ _, _ = fmt.Fprint(w, `{"success":true,"data":{"result":"ok"}}`)
+ default:
+ http.Error(w, "unexpected path: "+req.URL.Path, http.StatusNotFound)
+ }
+ }))
+ defer server.Close()
+ defer releaseEnd()
+
+ r := newStagehandRuntime(time.Hour, 1, time.Minute)
+ defer r.Close()
+ req := RunTaskRequest{Instruction: "do it", ModelName: "openai/gpt-4o", APIKey: "key"}
+ counted := &countedStagehandOption{RequestOption: option.WithBaseURL(server.URL)}
+ entry := &stagehandClientEntry{
+ client: stagehand.NewClient(counted, option.WithMaxRetries(0)),
+ }
+ entry.lastUsedAt.Store(time.Now().UnixNano())
+ r.cache.Store(cacheKey(req), entry)
+ done := make(chan error, 1)
+ go func() {
+ if operation == "task" {
+ _, err := r.RunTask(t.Context(), req)
+ done <- err
+ } else {
+ _, err := r.RunExtract(t.Context(), RunExtractRequest{
+ Instruction: req.Instruction, ModelName: req.ModelName, APIKey: req.APIKey,
+ Schema: map[string]any{"type": "string"},
+ })
+ done <- err
+ }
+ }()
+ select {
+ case <-endStarted:
+ case err := <-done:
+ t.Fatalf("operation completed before End: %v", err)
+ case <-time.After(3 * time.Second):
+ t.Fatal("Sessions.End did not start")
+ }
+ if eviction == "ttl" {
+ entry.lastUsedAt.Store(time.Now().Add(-2 * time.Hour).UnixNano())
+ r.evictExpired()
+ } else if eviction == "lru" {
+ _, _ = r.clientFor(RunTaskRequest{ModelName: req.ModelName, APIKey: "other"})
+ } else {
+ _ = r.Close()
+ }
+ if _, ok := r.cache.Load(cacheKey(req)); ok {
+ t.Fatal("entry was not evicted")
+ }
+ if got := counted.closes.Load(); got != 0 {
+ t.Fatalf("client closed before Sessions.End completed: %d", got)
+ }
+ releaseEnd()
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatalf("operation: %v", err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("operation did not finish")
+ }
+ r.Close()
+ if got := counted.closes.Load(); got != 1 {
+ t.Fatalf("client Close calls = %d, want 1", got)
+ }
+ })
+ }
+ }
+}
+
+func TestStagehandRuntime_RetiredClientWaitsForLastLease(t *testing.T) {
+ r := newStagehandRuntime(time.Hour, 0, time.Minute)
+ defer r.Close()
+ req := RunTaskRequest{ModelName: "openai/gpt-4o", APIKey: "key"}
+ counted := &countedStagehandOption{RequestOption: option.WithBaseURL("http://unused.test")}
+ entry := &stagehandClientEntry{client: stagehand.NewClient(counted)}
+ r.cache.Store(cacheKey(req), entry)
+
+ _, releaseFirst, err := r.leaseClient(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, releaseLast, err := r.leaseClient(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ entry.lastUsedAt.Store(time.Now().Add(-2 * time.Hour).UnixNano())
+ r.evictExpired()
+ _, releaseNew, err := r.leaseClient(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ releaseNew()
+ if v, _ := r.cache.Load(cacheKey(req)); v == entry {
+ t.Fatal("new acquisition reused a retired entry")
+ }
+ releaseFirst()
+ if got := counted.closes.Load(); got != 0 {
+ t.Fatalf("client closed with a remaining lease: %d", got)
+ }
+ var wg sync.WaitGroup
+ wg.Go(releaseLast)
+ for range 8 {
+ wg.Go(func() { _ = r.Close() })
+ }
+ wg.Wait()
+ if got := counted.closes.Load(); got != 1 {
+ t.Fatalf("Close calls = %d, want 1", got)
+ }
+ if _, _, err := r.leaseClient(req); err == nil {
+ t.Fatal("acquisition succeeded after runtime Close")
+ }
+}
+
// ---------------------------------------------------------------------------
// Env-driven config
// ---------------------------------------------------------------------------