RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(kvrocks): surface transport errors from SetNX and DistributedLock.Acquire (#20660) (#20774)
Harsh Kashyap805cd602ac5c
diff --git a/internal/engine/kvrocks/redis.go b/internal/engine/kvrocks/redis.goindex 673417edcf..146bb0350d 100644--- a/internal/engine/kvrocks/redis.go+++ b/internal/engine/kvrocks/redis.go@@ -369,16 +369,31 @@ func (r *Client) Set(ctx context.Context, key string, value string, exp time.Dur}// SetNX sets value only if key does not exist+//+// Deprecated: prefer SetNXErr, which surfaces transport errors instead of+// collapsing them into "lock held". Callers that only need a yes/no answer+// for an idempotent write can keep using SetNX; callers that retry on+// contention (notably DistributedLock.Acquire / SpinAcquire) should call+// SetNXErr so a connection outage is not mistaken for a held lock.func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) bool {+ ok, _ := r.SetNXErr(ctx, key, value, exp)+ return ok+}++// SetNXErr sets value only if key does not exist and reports whether the+// Redis/Kvrocks backend could be reached. ok=false with err=nil means the+// key already existed (lock contested); ok=false with err!=nil means the+// transport failed (caller should fail fast, not retry on contention).+func (r *Client) SetNXErr(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {if r.client == nil {- return false+ return false, nil}ok, err := r.client.SetNX(ctx, key, value, exp).Result()if err != nil {common.Warn("Redis SetNX error", zap.String("key", key), zap.Error(err))- return false+ return false, err}- return ok+ return ok, nil}// GetOrCreateKey atomically retrieves an existing key or creates a new one@@ -896,17 +911,26 @@ func NewDistributedLock(lockKey string, lockValue string, timeout time.Duration,}}-// Acquire acquires the lock-func (l *DistributedLock) Acquire(ctx context.Context) bool {+// Acquire attempts to take the lock. Returns (true, nil) on success,+// (false, nil) when the lock is held by someone else, and (false, err)+// when the underlying SETNX itself could not be issued (transport / auth).+// The third case is what used to be collapsed into "lock held" by the+// bool-only SetNX path, which made every DocumentService.WithDocument-+// MetadataLock caller spin for its full 15 s budget during a Redis/+// Kvrocks outage instead of failing immediately.+func (l *DistributedLock) Acquire(ctx context.Context) (bool, error) {if l.client == nil {- return false+ return false, nil}// Delete if stalel.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)- return l.client.SetNX(ctx, l.lockKey, l.lockValue, l.timeout)+ return l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)}-// SpinAcquire keeps trying to acquire the lock+// SpinAcquire keeps trying to acquire the lock. Returns the underlying+// transport error instead of silently logging it via SetNX so an outage+// surfaces immediately; ctx.Err() still wins when the context is+// cancelled mid-poll.func (l *DistributedLock) SpinAcquire(ctx context.Context) error {for {select {@@ -914,7 +938,11 @@ func (l *DistributedLock) SpinAcquire(ctx context.Context) error {return ctx.Err()default:l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)- if l.client.SetNX(ctx, l.lockKey, l.lockValue, l.timeout) {+ ok, err := l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)+ if err != nil {+ return err+ }+ if ok {return nil}time.Sleep(10 * time.Second)diff --git a/internal/engine/kvrocks/redis_test.go b/internal/engine/kvrocks/redis_test.goindex 28cbccdea5..e38cd6dcb1 100644--- a/internal/engine/kvrocks/redis_test.go+++ b/internal/engine/kvrocks/redis_test.go@@ -18,6 +18,7 @@ package kvrocksimport ("context"+ "errors""testing""time"@@ -144,3 +145,128 @@ func TestEvalTokenBucketStrict_NilClient(t *testing.T) {t.Fatalf("expected allowed=false on nil client (fail-closed)")}}++// TestSetNXErr_AllowedWhenNew pins the happy path: SetNXErr reports+// (true, nil) on a brand-new key. Regression guard for issue #20660.+func TestSetNXErr_AllowedWhenNew(t *testing.T) {+ r, _ := newStrictTestClient(t)+ ctx := t.Context()++ ok, err := r.SetNXErr(ctx, "lock:test:new", "v", time.Minute)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !ok {+ t.Fatalf("expected ok=true on a fresh key")+ }+}++// TestSetNXErr_FalseOnContention confirms the second concurrent caller+// sees (false, nil), not (false, err). Only the transport-failure case+// should return an error - keep the two outcomes distinguishable so the+// DistributedLock wait-for-lock loop can retry on the former and fail+// fast on the latter.+func TestSetNXErr_FalseOnContention(t *testing.T) {+ r, _ := newStrictTestClient(t)+ ctx := t.Context()++ if ok, err := r.SetNXErr(ctx, "lock:test:contended", "first", time.Minute); err != nil || !ok {+ t.Fatalf("seed call: ok=%v err=%v", ok, err)+ }+ ok, err := r.SetNXErr(ctx, "lock:test:contended", "second", time.Minute)+ if err != nil {+ t.Fatalf("contention should NOT surface a transport error, got %v", err)+ }+ if ok {+ t.Fatalf("expected ok=false on a held key")+ }+}++// TestSetNXErr_TransportErrorFailsFast is the core regression for+// #20660. With miniredis closed, the underlying SETNX cannot reach the+// server; SetNXErr must surface that as a non-nil error instead of+// returning false and forcing every distributed-lock caller to spin for+// its full timeout budget. Mirrors the EvalTokenBucketStrict+// fail-closed contract in the same package.+func TestSetNXErr_TransportErrorFailsFast(t *testing.T) {+ r, mr := newStrictTestClient(t)+ mr.Close()++ ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)+ defer cancel()++ ok, err := r.SetNXErr(ctx, "lock:test:down", "v", time.Minute)+ if err == nil {+ t.Fatalf("expected transport error, got nil (ok=%v)", ok)+ }+ if ok {+ t.Fatalf("expected ok=false on transport failure (fail-closed)")+ }+}++// TestDistributedLock_AcquireTransportError mirrors the issue body:+// DistributedLock.Acquire must surface the underlying transport error+// rather than collapsing it into (false, nil), otherwise every polling+// caller (the two withDDLLock waitForLock loops and any future spin+// caller) will treat "Redis is down" as "lock held by someone else".+func TestDistributedLock_AcquireTransportError(t *testing.T) {+ r, mr := newStrictTestClient(t)+ mr.Close()+ // Bypass NewDistributedLock's globalClient guard so we can drive+ // Acquire against a known-down client.+ lock := &DistributedLock{+ client: r,+ lockKey: "lock:test:down",+ lockValue: "v",+ timeout: time.Minute,+ blockingTimeout: 0,+ }++ ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)+ defer cancel()++ acquired, err := lock.Acquire(ctx)+ if err == nil {+ t.Fatalf("expected transport error from Acquire, got nil (acquired=%v)", acquired)+ }+ if acquired {+ t.Fatalf("expected acquired=false on transport failure (fail-closed)")+ }+}++// TestDistributedLock_SpinAcquireSurfacesTransportError confirms the+// second regression the issue called out: SpinAcquire used to swallow+// the underlying SetNX error and only return ctx.Err() at the next+// select-tick. With Redis down, SpinAcquire must return the transport+// error directly so the metadata-write path can fail fast (target:+// ~3s on a real outage, the go-redis connection-pool exhaustion time,+// rather than the 10s internal sleep plus caller budget).+//+// The 5s ctx is sized to give go-redis's 5-attempt connection pool+// exhaustion time to surface the connection-refused error before the+// test deadline cancels the call - otherwise the assertion would+// conflate "transport failure" with "ctx timed out first" (because+// go-redis's .Result() returns ctx.Err() when the parent context+// deadline beats the dial to the failed endpoint).+func TestDistributedLock_SpinAcquireSurfacesTransportError(t *testing.T) {+ r, mr := newStrictTestClient(t)+ mr.Close()+ lock := &DistributedLock{+ client: r,+ lockKey: "lock:test:down",+ lockValue: "v",+ timeout: time.Minute,+ blockingTimeout: 0,+ }++ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)+ defer cancel()++ err := lock.SpinAcquire(ctx)+ if err == nil {+ t.Fatalf("expected transport error from SpinAcquire, got nil")+ }+ if errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("SpinAcquire returned ctx.Err() (deadline) instead of the transport error: %v", err)+ }+}diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.goindex ac07091b69..27adf734c8 100644--- a/internal/engine/oceanbase/schema.go+++ b/internal/engine/oceanbase/schema.go@@ -326,28 +326,38 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()}timeout := time.Duration(timeoutSeconds) * time.Seconddistributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)- if distributed != nil && !distributed.Acquire(ctx) {- deadline := time.NewTimer(timeout)- defer deadline.Stop()- ticker := time.NewTicker(time.Second)- defer ticker.Stop()- waitForLock:- for {- select {- case <-ctx.Done():- return ctx.Err()- case <-deadline.C:- return fmt.Errorf("timeout waiting for DDL %s", lockName)- case <-ticker.C:- exists, err = check()- if err != nil {- return err- }- if exists {- return nil- }- if distributed.Acquire(ctx) {- break waitForLock+ if distributed != nil {+ acquired, err := distributed.Acquire(ctx)+ if err != nil {+ return err+ }+ if !acquired {+ deadline := time.NewTimer(timeout)+ defer deadline.Stop()+ ticker := time.NewTicker(time.Second)+ defer ticker.Stop()+ waitForLock:+ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-deadline.C:+ return fmt.Errorf("timeout waiting for DDL %s", lockName)+ case <-ticker.C:+ exists, err = check()+ if err != nil {+ return err+ }+ if exists {+ return nil+ }+ got, err := distributed.Acquire(ctx)+ if err != nil {+ return err+ }+ if got {+ break waitForLock+ }}}}diff --git a/internal/engine/vastbase/schema.go b/internal/engine/vastbase/schema.goindex ff6e8e2273..1f43aad026 100644--- a/internal/engine/vastbase/schema.go+++ b/internal/engine/vastbase/schema.go@@ -427,28 +427,38 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()timeout := 60 * time.Seconddistributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)- if distributed != nil && !distributed.Acquire(ctx) {- deadline := time.NewTimer(timeout)- defer deadline.Stop()- ticker := time.NewTicker(time.Second)- defer ticker.Stop()- waitForLock:- for {- select {- case <-ctx.Done():- return ctx.Err()- case <-deadline.C:- return fmt.Errorf("timeout waiting for DDL %s", lockName)- case <-ticker.C:- exists, err = check()- if err != nil {- return err- }- if exists {- return nil- }- if distributed.Acquire(ctx) {- break waitForLock+ if distributed != nil {+ acquired, err := distributed.Acquire(ctx)+ if err != nil {+ return err+ }+ if !acquired {+ deadline := time.NewTimer(timeout)+ defer deadline.Stop()+ ticker := time.NewTicker(time.Second)+ defer ticker.Stop()+ waitForLock:+ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-deadline.C:+ return fmt.Errorf("timeout waiting for DDL %s", lockName)+ case <-ticker.C:+ exists, err = check()+ if err != nil {+ return err+ }+ if exists {+ return nil+ }+ got, err := distributed.Acquire(ctx)+ if err != nil {+ return err+ }+ if got {+ break waitForLock+ }}}}diff --git a/internal/syncer/connector_lock.go b/internal/syncer/connector_lock.goindex 98b2f26bc5..6e06cf256c 100644--- a/internal/syncer/connector_lock.go+++ b/internal/syncer/connector_lock.go@@ -23,6 +23,8 @@ import ("ragflow/internal/engine/kvrocks""sync""time"++ "go.uber.org/zap")// ConnectorLocker serializes work for one connector and knowledge base.@@ -67,7 +69,19 @@ func (l *ConnectorLock) TryLock(connectorID, kbID string) (ConnectorLockLease, bif client := kvrocks.Get(); client != nil {lock := kvrocks.NewDistributedLock(key, l.holder, connectorLockTTL, 0)- if lock == nil || !lock.Acquire(context.Background()) {+ if lock != nil {+ acquired, err := lock.Acquire(context.Background())+ if err != nil {+ common.Warn("connector lock acquire transport error",+ zap.String("key", key), zap.Error(err))+ }+ if !acquired {+ l.mu.Lock()+ delete(l.local, key)+ l.mu.Unlock()+ return ConnectorLockLease{}, false+ }+ } else {l.mu.Lock()delete(l.local, key)l.mu.Unlock()