Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(kvrocks): surface transport errors from SetNX and DistributedLock.Acquire (#20660) (#20774)

Harsh Kashyap805cd602ac5c
diff --git a/internal/engine/kvrocks/redis.go b/internal/engine/kvrocks/redis.go
index 673417edcf..146bb0350d 100644
--- a/internal/engine/kvrocks/redis.go
+++ b/internal/engine/kvrocks/redis.go
@@ -369,16 +369,31 @@ func (r *Client) Set(ctx context.Context, key string, value string, exp time.Dur
}
// SetNX sets value only if key does not exist
+//
+// Deprecated: prefer SetNXErr, which surfaces transport errors instead of
+// collapsing them into "lock held". Callers that only need a yes/no answer
+// for an idempotent write can keep using SetNX; callers that retry on
+// contention (notably DistributedLock.Acquire / SpinAcquire) should call
+// SetNXErr so a connection outage is not mistaken for a held lock.
func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) bool {
+ ok, _ := r.SetNXErr(ctx, key, value, exp)
+ return ok
+}
+
+// SetNXErr sets value only if key does not exist and reports whether the
+// Redis/Kvrocks backend could be reached. ok=false with err=nil means the
+// key already existed (lock contested); ok=false with err!=nil means the
+// transport failed (caller should fail fast, not retry on contention).
+func (r *Client) SetNXErr(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {
if r.client == nil {
- return false
+ return false, nil
}
ok, err := r.client.SetNX(ctx, key, value, exp).Result()
if err != nil {
common.Warn("Redis SetNX error", zap.String("key", key), zap.Error(err))
- return false
+ return false, err
}
- return ok
+ return ok, nil
}
// GetOrCreateKey atomically retrieves an existing key or creates a new one
@@ -896,17 +911,26 @@ func NewDistributedLock(lockKey string, lockValue string, timeout time.Duration,
}
}
-// Acquire acquires the lock
-func (l *DistributedLock) Acquire(ctx context.Context) bool {
+// Acquire attempts to take the lock. Returns (true, nil) on success,
+// (false, nil) when the lock is held by someone else, and (false, err)
+// when the underlying SETNX itself could not be issued (transport / auth).
+// The third case is what used to be collapsed into "lock held" by the
+// bool-only SetNX path, which made every DocumentService.WithDocument-
+// MetadataLock caller spin for its full 15 s budget during a Redis/
+// Kvrocks outage instead of failing immediately.
+func (l *DistributedLock) Acquire(ctx context.Context) (bool, error) {
if l.client == nil {
- return false
+ return false, nil
}
// Delete if stale
l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)
- return l.client.SetNX(ctx, l.lockKey, l.lockValue, l.timeout)
+ return l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)
}
-// SpinAcquire keeps trying to acquire the lock
+// SpinAcquire keeps trying to acquire the lock. Returns the underlying
+// transport error instead of silently logging it via SetNX so an outage
+// surfaces immediately; ctx.Err() still wins when the context is
+// cancelled mid-poll.
func (l *DistributedLock) SpinAcquire(ctx context.Context) error {
for {
select {
@@ -914,7 +938,11 @@ func (l *DistributedLock) SpinAcquire(ctx context.Context) error {
return ctx.Err()
default:
l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)
- if l.client.SetNX(ctx, l.lockKey, l.lockValue, l.timeout) {
+ ok, err := l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)
+ if err != nil {
+ return err
+ }
+ if ok {
return nil
}
time.Sleep(10 * time.Second)
diff --git a/internal/engine/kvrocks/redis_test.go b/internal/engine/kvrocks/redis_test.go
index 28cbccdea5..e38cd6dcb1 100644
--- a/internal/engine/kvrocks/redis_test.go
+++ b/internal/engine/kvrocks/redis_test.go
@@ -18,6 +18,7 @@ package kvrocks
import (
"context"
+ "errors"
"testing"
"time"
@@ -144,3 +145,128 @@ func TestEvalTokenBucketStrict_NilClient(t *testing.T) {
t.Fatalf("expected allowed=false on nil client (fail-closed)")
}
}
+
+// TestSetNXErr_AllowedWhenNew pins the happy path: SetNXErr reports
+// (true, nil) on a brand-new key. Regression guard for issue #20660.
+func TestSetNXErr_AllowedWhenNew(t *testing.T) {
+ r, _ := newStrictTestClient(t)
+ ctx := t.Context()
+
+ ok, err := r.SetNXErr(ctx, "lock:test:new", "v", time.Minute)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !ok {
+ t.Fatalf("expected ok=true on a fresh key")
+ }
+}
+
+// TestSetNXErr_FalseOnContention confirms the second concurrent caller
+// sees (false, nil), not (false, err). Only the transport-failure case
+// should return an error - keep the two outcomes distinguishable so the
+// DistributedLock wait-for-lock loop can retry on the former and fail
+// fast on the latter.
+func TestSetNXErr_FalseOnContention(t *testing.T) {
+ r, _ := newStrictTestClient(t)
+ ctx := t.Context()
+
+ if ok, err := r.SetNXErr(ctx, "lock:test:contended", "first", time.Minute); err != nil || !ok {
+ t.Fatalf("seed call: ok=%v err=%v", ok, err)
+ }
+ ok, err := r.SetNXErr(ctx, "lock:test:contended", "second", time.Minute)
+ if err != nil {
+ t.Fatalf("contention should NOT surface a transport error, got %v", err)
+ }
+ if ok {
+ t.Fatalf("expected ok=false on a held key")
+ }
+}
+
+// TestSetNXErr_TransportErrorFailsFast is the core regression for
+// #20660. With miniredis closed, the underlying SETNX cannot reach the
+// server; SetNXErr must surface that as a non-nil error instead of
+// returning false and forcing every distributed-lock caller to spin for
+// its full timeout budget. Mirrors the EvalTokenBucketStrict
+// fail-closed contract in the same package.
+func TestSetNXErr_TransportErrorFailsFast(t *testing.T) {
+ r, mr := newStrictTestClient(t)
+ mr.Close()
+
+ ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
+ defer cancel()
+
+ ok, err := r.SetNXErr(ctx, "lock:test:down", "v", time.Minute)
+ if err == nil {
+ t.Fatalf("expected transport error, got nil (ok=%v)", ok)
+ }
+ if ok {
+ t.Fatalf("expected ok=false on transport failure (fail-closed)")
+ }
+}
+
+// TestDistributedLock_AcquireTransportError mirrors the issue body:
+// DistributedLock.Acquire must surface the underlying transport error
+// rather than collapsing it into (false, nil), otherwise every polling
+// caller (the two withDDLLock waitForLock loops and any future spin
+// caller) will treat "Redis is down" as "lock held by someone else".
+func TestDistributedLock_AcquireTransportError(t *testing.T) {
+ r, mr := newStrictTestClient(t)
+ mr.Close()
+ // Bypass NewDistributedLock's globalClient guard so we can drive
+ // Acquire against a known-down client.
+ lock := &DistributedLock{
+ client: r,
+ lockKey: "lock:test:down",
+ lockValue: "v",
+ timeout: time.Minute,
+ blockingTimeout: 0,
+ }
+
+ ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
+ defer cancel()
+
+ acquired, err := lock.Acquire(ctx)
+ if err == nil {
+ t.Fatalf("expected transport error from Acquire, got nil (acquired=%v)", acquired)
+ }
+ if acquired {
+ t.Fatalf("expected acquired=false on transport failure (fail-closed)")
+ }
+}
+
+// TestDistributedLock_SpinAcquireSurfacesTransportError confirms the
+// second regression the issue called out: SpinAcquire used to swallow
+// the underlying SetNX error and only return ctx.Err() at the next
+// select-tick. With Redis down, SpinAcquire must return the transport
+// error directly so the metadata-write path can fail fast (target:
+// ~3s on a real outage, the go-redis connection-pool exhaustion time,
+// rather than the 10s internal sleep plus caller budget).
+//
+// The 5s ctx is sized to give go-redis's 5-attempt connection pool
+// exhaustion time to surface the connection-refused error before the
+// test deadline cancels the call - otherwise the assertion would
+// conflate "transport failure" with "ctx timed out first" (because
+// go-redis's .Result() returns ctx.Err() when the parent context
+// deadline beats the dial to the failed endpoint).
+func TestDistributedLock_SpinAcquireSurfacesTransportError(t *testing.T) {
+ r, mr := newStrictTestClient(t)
+ mr.Close()
+ lock := &DistributedLock{
+ client: r,
+ lockKey: "lock:test:down",
+ lockValue: "v",
+ timeout: time.Minute,
+ blockingTimeout: 0,
+ }
+
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancel()
+
+ err := lock.SpinAcquire(ctx)
+ if err == nil {
+ t.Fatalf("expected transport error from SpinAcquire, got nil")
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("SpinAcquire returned ctx.Err() (deadline) instead of the transport error: %v", err)
+ }
+}
diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.go
index ac07091b69..27adf734c8 100644
--- a/internal/engine/oceanbase/schema.go
+++ b/internal/engine/oceanbase/schema.go
@@ -326,28 +326,38 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()
}
timeout := time.Duration(timeoutSeconds) * time.Second
distributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)
- if distributed != nil && !distributed.Acquire(ctx) {
- deadline := time.NewTimer(timeout)
- defer deadline.Stop()
- ticker := time.NewTicker(time.Second)
- defer ticker.Stop()
- waitForLock:
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-deadline.C:
- return fmt.Errorf("timeout waiting for DDL %s", lockName)
- case <-ticker.C:
- exists, err = check()
- if err != nil {
- return err
- }
- if exists {
- return nil
- }
- if distributed.Acquire(ctx) {
- break waitForLock
+ if distributed != nil {
+ acquired, err := distributed.Acquire(ctx)
+ if err != nil {
+ return err
+ }
+ if !acquired {
+ deadline := time.NewTimer(timeout)
+ defer deadline.Stop()
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ waitForLock:
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-deadline.C:
+ return fmt.Errorf("timeout waiting for DDL %s", lockName)
+ case <-ticker.C:
+ exists, err = check()
+ if err != nil {
+ return err
+ }
+ if exists {
+ return nil
+ }
+ got, err := distributed.Acquire(ctx)
+ if err != nil {
+ return err
+ }
+ if got {
+ break waitForLock
+ }
}
}
}
diff --git a/internal/engine/vastbase/schema.go b/internal/engine/vastbase/schema.go
index ff6e8e2273..1f43aad026 100644
--- a/internal/engine/vastbase/schema.go
+++ b/internal/engine/vastbase/schema.go
@@ -427,28 +427,38 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()
timeout := 60 * time.Second
distributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)
- if distributed != nil && !distributed.Acquire(ctx) {
- deadline := time.NewTimer(timeout)
- defer deadline.Stop()
- ticker := time.NewTicker(time.Second)
- defer ticker.Stop()
- waitForLock:
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-deadline.C:
- return fmt.Errorf("timeout waiting for DDL %s", lockName)
- case <-ticker.C:
- exists, err = check()
- if err != nil {
- return err
- }
- if exists {
- return nil
- }
- if distributed.Acquire(ctx) {
- break waitForLock
+ if distributed != nil {
+ acquired, err := distributed.Acquire(ctx)
+ if err != nil {
+ return err
+ }
+ if !acquired {
+ deadline := time.NewTimer(timeout)
+ defer deadline.Stop()
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ waitForLock:
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-deadline.C:
+ return fmt.Errorf("timeout waiting for DDL %s", lockName)
+ case <-ticker.C:
+ exists, err = check()
+ if err != nil {
+ return err
+ }
+ if exists {
+ return nil
+ }
+ got, err := distributed.Acquire(ctx)
+ if err != nil {
+ return err
+ }
+ if got {
+ break waitForLock
+ }
}
}
}
diff --git a/internal/syncer/connector_lock.go b/internal/syncer/connector_lock.go
index 98b2f26bc5..6e06cf256c 100644
--- a/internal/syncer/connector_lock.go
+++ b/internal/syncer/connector_lock.go
@@ -23,6 +23,8 @@ import (
"ragflow/internal/engine/kvrocks"
"sync"
"time"
+
+ "go.uber.org/zap"
)
// ConnectorLocker serializes work for one connector and knowledge base.
@@ -67,7 +69,19 @@ func (l *ConnectorLock) TryLock(connectorID, kbID string) (ConnectorLockLease, b
if client := kvrocks.Get(); client != nil {
lock := kvrocks.NewDistributedLock(key, l.holder, connectorLockTTL, 0)
- if lock == nil || !lock.Acquire(context.Background()) {
+ if lock != nil {
+ acquired, err := lock.Acquire(context.Background())
+ if err != nil {
+ common.Warn("connector lock acquire transport error",
+ zap.String("key", key), zap.Error(err))
+ }
+ if !acquired {
+ l.mu.Lock()
+ delete(l.local, key)
+ l.mu.Unlock()
+ return ConnectorLockLease{}, false
+ }
+ } else {
l.mu.Lock()
delete(l.local, key)
l.mu.Unlock()