Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(kvrocks): propagate SetNX errors (#20746)

eleven914e2d1c25e19e1
diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.go
index dac007038a..8c946ae9dc 100644
--- a/internal/dao/sync_task.go
+++ b/internal/dao/sync_task.go
@@ -278,28 +278,60 @@ func (d *SyncTaskDAO) IsTaskCanceled(ctx context.Context, taskID string) (bool,
return task.Status == SyncStatusCancel, nil
}
+// GetTaskStatus returns the current status of a sync task. found is false when
+// the task no longer exists, allowing consumers to acknowledge stale messages.
+func (d *SyncTaskDAO) GetTaskStatus(ctx context.Context, taskID string) (status string, found bool, err error) {
+ var task entity.SyncLogs
+ err = d.db.WithContext(ctx).Select("status").Where("id = ?", taskID).Take(&task).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return "", false, nil
+ }
+ if err != nil {
+ return "", false, err
+ }
+ return task.Status, true, nil
+}
+
// MarkConnectorRunning marks a connector running.
func (d *SyncTaskDAO) MarkConnectorRunning(ctx context.Context, connectorID string) error {
return d.db.WithContext(ctx).Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusRunning).Error
}
-// RescheduleClaimed puts a claimed task back into schedule state.
-func (d *SyncTaskDAO) RescheduleClaimed(ctx context.Context, taskID string) error {
- return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+// RescheduleClaimed puts a claimed task back into schedule state. When
+// startedAt is non-nil, only the worker that owns that exact claim may restore
+// it, so a delayed retry cannot overwrite a newer claim.
+func (d *SyncTaskDAO) RescheduleClaimed(ctx context.Context, taskID string, startedAt *time.Time) (bool, error) {
+ var restored bool
+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task entity.SyncLogs
- if err := tx.WithContext(ctx).Where("id = ? AND status = ?", taskID, SyncStatusRunning).First(&task).Error; err != nil {
+ query := tx.WithContext(ctx).Where("id = ? AND status = ?", taskID, SyncStatusRunning)
+ if startedAt != nil {
+ query = query.Where("time_started = ?", *startedAt)
+ }
+ if err := query.First(&task).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return err
}
- if err := tx.Model(&entity.SyncLogs{}).
- Where("id = ? AND status = ?", taskID, SyncStatusRunning).
- Update("status", SyncStatusSchedule).Error; err != nil {
+ update := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", taskID, SyncStatusRunning)
+ if startedAt != nil {
+ update = update.Where("time_started = ?", *startedAt)
+ }
+ result := update.Update("status", SyncStatusSchedule)
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected == 0 {
+ return nil
+ }
+ if err := tx.Model(&entity.Connector{}).Where("id = ? AND status = ?", task.ConnectorID, SyncStatusRunning).Update("status", SyncStatusSchedule).Error; err != nil {
return err
}
- return tx.Model(&entity.Connector{}).Where("id = ? AND status = ?", task.ConnectorID, SyncStatusRunning).Update("status", SyncStatusSchedule).Error
+ restored = true
+ return nil
})
+ return restored, err
}
// FailTask marks a task failed without advancing its poll waterline.
diff --git a/internal/dao/sync_task_test.go b/internal/dao/sync_task_test.go
index 28a6f99383..aa9052f0ca 100644
--- a/internal/dao/sync_task_test.go
+++ b/internal/dao/sync_task_test.go
@@ -21,6 +21,7 @@ import (
"fmt"
"strings"
"testing"
+ "time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
@@ -113,6 +114,50 @@ func insertRunningSyncTask(t *testing.T, db *gorm.DB, taskID string, errorCount,
}
}
+func TestRescheduleClaimedPreservesNewerClaim(t *testing.T) {
+ db := setupSyncTaskTestDB(t)
+ if err := db.AutoMigrate(&entity.Connector{}); err != nil {
+ t.Fatalf("migrate connector: %v", err)
+ }
+ dao := NewSyncTaskDAO(db)
+ staleStartedAt := time.Now().Truncate(time.Millisecond)
+ currentStartedAt := staleStartedAt.Add(time.Second)
+ if err := db.Create(&entity.SyncLogs{
+ ID: "task-1",
+ ConnectorID: "conn-1",
+ KbID: "kb-1",
+ TaskType: TaskTypeSync,
+ Status: SyncStatusRunning,
+ TimeStarted: &currentStartedAt,
+ }).Error; err != nil {
+ t.Fatalf("insert task: %v", err)
+ }
+
+ restored, err := dao.RescheduleClaimed(t.Context(), "task-1", &staleStartedAt)
+ if err != nil {
+ t.Fatalf("reschedule stale claim: %v", err)
+ }
+ if restored {
+ t.Fatal("stale claim restored a newer claim")
+ }
+
+ var task entity.SyncLogs
+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != SyncStatusRunning {
+ t.Fatalf("status = %s, want running", task.Status)
+ }
+
+ restored, err = dao.RescheduleClaimed(t.Context(), "task-1", &currentStartedAt)
+ if err != nil {
+ t.Fatalf("reschedule current claim: %v", err)
+ }
+ if !restored {
+ t.Fatal("current claim was not restored")
+ }
+}
+
// TestHandleTransientFailureUsesClassScopedRetryBudget verifies the retry
// comparison uses a counter scoped to the current error class: failures under
// one class must not consume another class's budget, and ErrorCount keeps the
diff --git a/internal/engine/kvrocks/redis.go b/internal/engine/kvrocks/redis.go
index 146bb0350d..6b2a006800 100644
--- a/internal/engine/kvrocks/redis.go
+++ b/internal/engine/kvrocks/redis.go
@@ -368,32 +368,24 @@ func (r *Client) Set(ctx context.Context, key string, value string, exp time.Dur
return true
}
-// SetNX sets value only if key does not exist
-//
-// Deprecated: prefer SetNXErr, which surfaces transport errors instead of
-// collapsing them into "lock held". Callers that only need a yes/no answer
-// for an idempotent write can keep using SetNX; callers that retry on
-// contention (notably DistributedLock.Acquire / SpinAcquire) should call
-// SetNXErr so a connection outage is not mistaken for a held lock.
-func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) bool {
- ok, _ := r.SetNXErr(ctx, key, value, exp)
- return ok
-}
-
-// SetNXErr sets value only if key does not exist and reports whether the
-// Redis/Kvrocks backend could be reached. ok=false with err=nil means the
-// key already existed (lock contested); ok=false with err!=nil means the
-// transport failed (caller should fail fast, not retry on contention).
-func (r *Client) SetNXErr(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {
- if r.client == nil {
+// SetNX sets value only if key does not exist.
+// It returns false with a nil error when the key already exists.
+func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {
+ if r == nil || r.client == nil {
+ return false, errors.New("kvrocks client is not initialized")
+ }
+ result, err := r.client.SetArgs(ctx, key, value, redis.SetArgs{
+ Mode: "NX",
+ TTL: exp,
+ }).Result()
+ if errors.Is(err, redis.Nil) {
return false, nil
}
- ok, err := r.client.SetNX(ctx, key, value, exp).Result()
if err != nil {
common.Warn("Redis SetNX error", zap.String("key", key), zap.Error(err))
- return false, err
+ return false, fmt.Errorf("set key %q if absent: %w", key, err)
}
- return ok, nil
+ return result == "OK", nil
}
// GetOrCreateKey atomically retrieves an existing key or creates a new one
@@ -835,17 +827,20 @@ func (r *Client) QueueInfo(ctx context.Context, queue, groupName string) (map[st
return nil, nil
}
-// DeleteIfEqual deletes key if its value equals expected value (atomic)
-func (r *Client) DeleteIfEqual(ctx context.Context, key, expectedValue string) bool {
- if r.client == nil {
- return false
+// deleteIfEqual deletes key if its value equals expected value atomically.
+func (r *Client) deleteIfEqual(ctx context.Context, key, expectedValue string) (bool, error) {
+ if r == nil || r.client == nil {
+ return false, errors.New("kvrocks client is not initialized")
+ }
+ if r.luaDeleteIfEqual == nil {
+ return false, errors.New("kvrocks delete-if-equal script is not initialized")
}
- result, err := r.luaDeleteIfEqual.Run(ctx, r.client, []string{key}, expectedValue).Result()
+ result, err := r.luaDeleteIfEqual.Run(ctx, r.client, []string{key}, expectedValue).Int64()
if err != nil {
- common.Warn("Redis DeleteIfEqual error", zap.Error(err))
- return false
+ common.Warn("Redis DeleteIfEqual error", zap.String("key", key), zap.Error(err))
+ return false, fmt.Errorf("delete key %q if value matches: %w", key, err)
}
- return result.(int64) == 1
+ return result == 1, nil
}
// Delete deletes a key
@@ -911,20 +906,17 @@ func NewDistributedLock(lockKey string, lockValue string, timeout time.Duration,
}
}
-// Acquire attempts to take the lock. Returns (true, nil) on success,
-// (false, nil) when the lock is held by someone else, and (false, err)
-// when the underlying SETNX itself could not be issued (transport / auth).
-// The third case is what used to be collapsed into "lock held" by the
-// bool-only SetNX path, which made every DocumentService.WithDocument-
-// MetadataLock caller spin for its full 15 s budget during a Redis/
-// Kvrocks outage instead of failing immediately.
+// Acquire acquires the lock. It returns false with a nil error when another
+// owner holds the lock.
func (l *DistributedLock) Acquire(ctx context.Context) (bool, error) {
- if l.client == nil {
- return false, nil
+ if l == nil || l.client == nil {
+ return false, errors.New("distributed lock client is not initialized")
+ }
+ // Remove a lock left by this owner before trying to acquire it again.
+ if _, err := l.client.deleteIfEqual(ctx, l.lockKey, l.lockValue); err != nil {
+ return false, err
}
- // Delete if stale
- l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)
- return l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)
+ return l.client.SetNX(ctx, l.lockKey, l.lockValue, l.timeout)
}
// SpinAcquire keeps trying to acquire the lock. Returns the underlying
@@ -937,12 +929,11 @@ func (l *DistributedLock) SpinAcquire(ctx context.Context) error {
case <-ctx.Done():
return ctx.Err()
default:
- l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)
- ok, err := l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)
+ acquired, err := l.Acquire(ctx)
if err != nil {
return err
}
- if ok {
+ if acquired {
return nil
}
time.Sleep(10 * time.Second)
@@ -952,10 +943,11 @@ func (l *DistributedLock) SpinAcquire(ctx context.Context) error {
// Release releases the lock
func (l *DistributedLock) Release(ctx context.Context) bool {
- if l.client == nil {
+ if l == nil || l.client == nil {
return false
}
- return l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)
+ released, err := l.client.deleteIfEqual(ctx, l.lockKey, l.lockValue)
+ return err == nil && released
}
// TokenBucket token bucket rate limiter
diff --git a/internal/engine/kvrocks/redis_test.go b/internal/engine/kvrocks/redis_test.go
index e38cd6dcb1..1e638b1876 100644
--- a/internal/engine/kvrocks/redis_test.go
+++ b/internal/engine/kvrocks/redis_test.go
@@ -18,7 +18,6 @@ package kvrocks
import (
"context"
- "errors"
"testing"
"time"
@@ -146,127 +145,96 @@ func TestEvalTokenBucketStrict_NilClient(t *testing.T) {
}
}
-// TestSetNXErr_AllowedWhenNew pins the happy path: SetNXErr reports
-// (true, nil) on a brand-new key. Regression guard for issue #20660.
-func TestSetNXErr_AllowedWhenNew(t *testing.T) {
+func TestClientSetNXDistinguishesContention(t *testing.T) {
r, _ := newStrictTestClient(t)
ctx := t.Context()
- ok, err := r.SetNXErr(ctx, "lock:test:new", "v", time.Minute)
+ acquired, err := r.SetNX(ctx, "lock:document", "owner-1", time.Minute)
if err != nil {
- t.Fatalf("unexpected error: %v", err)
+ t.Fatalf("first SetNX returned error: %v", err)
}
- if !ok {
- t.Fatalf("expected ok=true on a fresh key")
+ if !acquired {
+ t.Fatal("first SetNX acquired = false, want true")
}
-}
-
-// TestSetNXErr_FalseOnContention confirms the second concurrent caller
-// sees (false, nil), not (false, err). Only the transport-failure case
-// should return an error - keep the two outcomes distinguishable so the
-// DistributedLock wait-for-lock loop can retry on the former and fail
-// fast on the latter.
-func TestSetNXErr_FalseOnContention(t *testing.T) {
- r, _ := newStrictTestClient(t)
- ctx := t.Context()
- if ok, err := r.SetNXErr(ctx, "lock:test:contended", "first", time.Minute); err != nil || !ok {
- t.Fatalf("seed call: ok=%v err=%v", ok, err)
- }
- ok, err := r.SetNXErr(ctx, "lock:test:contended", "second", time.Minute)
+ acquired, err = r.SetNX(ctx, "lock:document", "owner-2", time.Minute)
if err != nil {
- t.Fatalf("contention should NOT surface a transport error, got %v", err)
+ t.Fatalf("contended SetNX returned error: %v", err)
}
- if ok {
- t.Fatalf("expected ok=false on a held key")
+ if acquired {
+ t.Fatal("contended SetNX acquired = true, want false")
}
}
-// TestSetNXErr_TransportErrorFailsFast is the core regression for
-// #20660. With miniredis closed, the underlying SETNX cannot reach the
-// server; SetNXErr must surface that as a non-nil error instead of
-// returning false and forcing every distributed-lock caller to spin for
-// its full timeout budget. Mirrors the EvalTokenBucketStrict
-// fail-closed contract in the same package.
-func TestSetNXErr_TransportErrorFailsFast(t *testing.T) {
+func TestClientSetNXReturnsTransportError(t *testing.T) {
r, mr := newStrictTestClient(t)
mr.Close()
ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
defer cancel()
- ok, err := r.SetNXErr(ctx, "lock:test:down", "v", time.Minute)
+ acquired, err := r.SetNX(ctx, "lock:document", "owner-1", time.Minute)
if err == nil {
- t.Fatalf("expected transport error, got nil (ok=%v)", ok)
+ t.Fatal("SetNX error = nil, want transport error")
}
- if ok {
- t.Fatalf("expected ok=false on transport failure (fail-closed)")
+ if acquired {
+ t.Fatal("SetNX acquired = true on transport failure, want false")
}
}
-// TestDistributedLock_AcquireTransportError mirrors the issue body:
-// DistributedLock.Acquire must surface the underlying transport error
-// rather than collapsing it into (false, nil), otherwise every polling
-// caller (the two withDDLLock waitForLock loops and any future spin
-// caller) will treat "Redis is down" as "lock held by someone else".
-func TestDistributedLock_AcquireTransportError(t *testing.T) {
- r, mr := newStrictTestClient(t)
- mr.Close()
- // Bypass NewDistributedLock's globalClient guard so we can drive
- // Acquire against a known-down client.
- lock := &DistributedLock{
- client: r,
- lockKey: "lock:test:down",
- lockValue: "v",
- timeout: time.Minute,
- blockingTimeout: 0,
- }
-
- ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
- defer cancel()
+func TestClientSetNXReturnsErrorForNilClient(t *testing.T) {
+ var r *Client
- acquired, err := lock.Acquire(ctx)
+ acquired, err := r.SetNX(t.Context(), "lock:document", "owner-1", time.Minute)
if err == nil {
- t.Fatalf("expected transport error from Acquire, got nil (acquired=%v)", acquired)
+ t.Fatal("SetNX error = nil, want initialization error")
}
if acquired {
- t.Fatalf("expected acquired=false on transport failure (fail-closed)")
+ t.Fatal("SetNX acquired = true for nil client, want false")
}
}
-// TestDistributedLock_SpinAcquireSurfacesTransportError confirms the
-// second regression the issue called out: SpinAcquire used to swallow
-// the underlying SetNX error and only return ctx.Err() at the next
-// select-tick. With Redis down, SpinAcquire must return the transport
-// error directly so the metadata-write path can fail fast (target:
-// ~3s on a real outage, the go-redis connection-pool exhaustion time,
-// rather than the 10s internal sleep plus caller budget).
-//
-// The 5s ctx is sized to give go-redis's 5-attempt connection pool
-// exhaustion time to surface the connection-refused error before the
-// test deadline cancels the call - otherwise the assertion would
-// conflate "transport failure" with "ctx timed out first" (because
-// go-redis's .Result() returns ctx.Err() when the parent context
-// deadline beats the dial to the failed endpoint).
-func TestDistributedLock_SpinAcquireSurfacesTransportError(t *testing.T) {
+func TestDistributedLockSpinAcquireReturnsTransportError(t *testing.T) {
r, mr := newStrictTestClient(t)
mr.Close()
lock := &DistributedLock{
- client: r,
- lockKey: "lock:test:down",
- lockValue: "v",
- timeout: time.Minute,
- blockingTimeout: 0,
+ client: r,
+ lockKey: "lock:document",
+ lockValue: "owner-1",
+ timeout: time.Minute,
}
- ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
defer cancel()
- err := lock.SpinAcquire(ctx)
+ if err := lock.SpinAcquire(ctx); err == nil {
+ t.Fatal("SpinAcquire error = nil, want transport error")
+ }
+}
+
+func TestDistributedLockAcquireReturnsCleanupError(t *testing.T) {
+ r, _ := newStrictTestClient(t)
+ r.luaDeleteIfEqual = redis.NewScript(`return redis.error_reply("cleanup failed")`)
+ lock := &DistributedLock{
+ client: r,
+ lockKey: "lock:document",
+ lockValue: "owner-1",
+ timeout: time.Minute,
+ }
+
+ acquired, err := lock.Acquire(t.Context())
if err == nil {
- t.Fatalf("expected transport error from SpinAcquire, got nil")
+ t.Fatal("Acquire error = nil, want cleanup error")
+ }
+ if acquired {
+ t.Fatal("Acquire acquired = true on cleanup failure, want false")
+ }
+
+ exists, err := r.client.Exists(t.Context(), lock.lockKey).Result()
+ if err != nil {
+ t.Fatalf("check lock key after cleanup failure: %v", err)
}
- if errors.Is(err, context.DeadlineExceeded) {
- t.Fatalf("SpinAcquire returned ctx.Err() (deadline) instead of the transport error: %v", err)
+ if exists != 0 {
+ t.Fatal("Acquire attempted SetNX after cleanup failure")
}
}
diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.go
index 8f40d4bb5e..59e4536368 100644
--- a/internal/engine/oceanbase/schema.go
+++ b/internal/engine/oceanbase/schema.go
@@ -326,38 +326,39 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()
}
timeout := time.Duration(timeoutSeconds) * time.Second
distributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)
+ acquired := false
if distributed != nil {
- acquired, err := distributed.Acquire(ctx)
+ acquired, err = distributed.Acquire(ctx)
if err != nil {
- return err
+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)
}
- if !acquired {
- deadline := time.NewTimer(timeout)
- defer deadline.Stop()
- ticker := time.NewTicker(time.Second)
- defer ticker.Stop()
- waitForLock:
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-deadline.C:
- return fmt.Errorf("timeout waiting for DDL %s", lockName)
- case <-ticker.C:
- exists, err = check()
- if err != nil {
- return err
- }
- if exists {
- return nil
- }
- got, err := distributed.Acquire(ctx)
- if err != nil {
- return err
- }
- if got {
- break waitForLock
- }
+ }
+ if distributed != nil && !acquired {
+ deadline := time.NewTimer(timeout)
+ defer deadline.Stop()
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ waitForLock:
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-deadline.C:
+ return fmt.Errorf("timeout waiting for DDL %s", lockName)
+ case <-ticker.C:
+ exists, err = check()
+ if err != nil {
+ return err
+ }
+ if exists {
+ return nil
+ }
+ acquired, err = distributed.Acquire(ctx)
+ if err != nil {
+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)
+ }
+ if acquired {
+ break waitForLock
}
}
}
diff --git a/internal/engine/vastbase/schema.go b/internal/engine/vastbase/schema.go
index 1b5cf1678a..f3f1fc71a8 100644
--- a/internal/engine/vastbase/schema.go
+++ b/internal/engine/vastbase/schema.go
@@ -427,38 +427,39 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()
timeout := 60 * time.Second
distributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)
+ acquired := false
if distributed != nil {
- acquired, err := distributed.Acquire(ctx)
+ acquired, err = distributed.Acquire(ctx)
if err != nil {
- return err
+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)
}
- if !acquired {
- deadline := time.NewTimer(timeout)
- defer deadline.Stop()
- ticker := time.NewTicker(time.Second)
- defer ticker.Stop()
- waitForLock:
- for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-deadline.C:
- return fmt.Errorf("timeout waiting for DDL %s", lockName)
- case <-ticker.C:
- exists, err = check()
- if err != nil {
- return err
- }
- if exists {
- return nil
- }
- got, err := distributed.Acquire(ctx)
- if err != nil {
- return err
- }
- if got {
- break waitForLock
- }
+ }
+ if distributed != nil && !acquired {
+ deadline := time.NewTimer(timeout)
+ defer deadline.Stop()
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ waitForLock:
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-deadline.C:
+ return fmt.Errorf("timeout waiting for DDL %s", lockName)
+ case <-ticker.C:
+ exists, err = check()
+ if err != nil {
+ return err
+ }
+ if exists {
+ return nil
+ }
+ acquired, err = distributed.Acquire(ctx)
+ if err != nil {
+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)
+ }
+ if acquired {
+ break waitForLock
}
}
}
diff --git a/internal/server/variable.go b/internal/server/variable.go
index 693cb53eaf..bfa11f6079 100644
--- a/internal/server/variable.go
+++ b/internal/server/variable.go
@@ -36,7 +36,7 @@ type Variables struct {
type VariableStore interface {
Get(ctx context.Context, key string) (string, error)
Set(ctx context.Context, key string, value string, exp time.Duration) bool
- SetNX(ctx context.Context, key string, value string, exp time.Duration) bool
+ SetNX(ctx context.Context, key string, value string, exp time.Duration) (bool, error)
}
var (
@@ -144,7 +144,11 @@ func GetOrCreateKey(ctx context.Context, store VariableStore, key string, newVal
common.Info("Generating new value for key", zap.String("key", key))
// Try to set with NX (only if not exists) - ensures atomicity
- if store.SetNX(ctx, key, newValue, SecretKeyTTL) {
+ created, err := store.SetNX(ctx, key, newValue, SecretKeyTTL)
+ if err != nil {
+ return "", fmt.Errorf("failed to create key %q: %w", key, err)
+ }
+ if created {
common.Info("New value stored successfully", zap.String("key", key))
return newValue, nil
}
diff --git a/internal/server/variable_test.go b/internal/server/variable_test.go
new file mode 100644
index 0000000000..842ac1c3a5
--- /dev/null
+++ b/internal/server/variable_test.go
@@ -0,0 +1,94 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package server
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+)
+
+type variableStoreStub struct {
+ value string
+ getErr error
+ setNX bool
+ setNXErr error
+}
+
+func (s *variableStoreStub) Get(context.Context, string) (string, error) {
+ return s.value, s.getErr
+}
+
+func (s *variableStoreStub) Set(context.Context, string, string, time.Duration) bool {
+ return true
+}
+
+func (s *variableStoreStub) SetNX(context.Context, string, string, time.Duration) (bool, error) {
+ return s.setNX, s.setNXErr
+}
+
+func TestGetOrCreateKeyReturnsSetNXError(t *testing.T) {
+ transportErr := errors.New("kvrocks unavailable")
+ store := &variableStoreStub{setNXErr: transportErr}
+
+ value, err := GetOrCreateKey(t.Context(), store, "ragflow:key", "generated")
+ if err == nil {
+ t.Fatal("GetOrCreateKey error = nil, want SetNX error")
+ }
+ if !errors.Is(err, transportErr) {
+ t.Fatalf("GetOrCreateKey error = %v, want wrapped transport error", err)
+ }
+ if value != "" {
+ t.Fatalf("GetOrCreateKey value = %q, want empty string", value)
+ }
+ if !strings.Contains(err.Error(), "ragflow:key") {
+ t.Fatalf("GetOrCreateKey error = %q, want key context", err)
+ }
+}
+
+func TestGetOrCreateKeyHandlesSetNXContention(t *testing.T) {
+ // The first read misses. After SetNX reports contention, the second read
+ // observes the value created by the competing process.
+ storeWithSequence := &sequenceVariableStore{values: []string{"", "competing"}}
+ value, err := GetOrCreateKey(t.Context(), storeWithSequence, "ragflow:key", "generated")
+ if err != nil {
+ t.Fatalf("GetOrCreateKey returned error: %v", err)
+ }
+ if value != "competing" {
+ t.Fatalf("GetOrCreateKey value = %q, want %q", value, "competing")
+ }
+}
+
+type sequenceVariableStore struct {
+ values []string
+}
+
+func (s *sequenceVariableStore) Get(context.Context, string) (string, error) {
+ value := s.values[0]
+ s.values = s.values[1:]
+ return value, nil
+}
+
+func (s *sequenceVariableStore) Set(context.Context, string, string, time.Duration) bool {
+ return true
+}
+
+func (s *sequenceVariableStore) SetNX(context.Context, string, string, time.Duration) (bool, error) {
+ return false, nil
+}
diff --git a/internal/syncer/connector_lock.go b/internal/syncer/connector_lock.go
index 6e06cf256c..79770d9d2f 100644
--- a/internal/syncer/connector_lock.go
+++ b/internal/syncer/connector_lock.go
@@ -23,13 +23,11 @@ import (
"ragflow/internal/engine/kvrocks"
"sync"
"time"
-
- "go.uber.org/zap"
)
// ConnectorLocker serializes work for one connector and knowledge base.
type ConnectorLocker interface {
- TryLock(connectorID, kbID string) (ConnectorLockLease, bool)
+ TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error)
Unlock(connectorID, kbID string)
}
@@ -54,45 +52,46 @@ func NewConnectorLock() *ConnectorLock {
}
// TryLock attempts to acquire the connector/KB lock without blocking.
-func (l *ConnectorLock) TryLock(connectorID, kbID string) (ConnectorLockLease, bool) {
+func (l *ConnectorLock) TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error) {
if l == nil {
- return ConnectorLockLease{}, false
+ return ConnectorLockLease{}, false, nil
}
key := connectorLockKey(connectorID, kbID)
l.mu.Lock()
if _, ok := l.local[key]; ok {
l.mu.Unlock()
- return ConnectorLockLease{}, false
+ return ConnectorLockLease{}, false, nil
}
l.local[key] = struct{}{}
l.mu.Unlock()
if client := kvrocks.Get(); client != nil {
lock := kvrocks.NewDistributedLock(key, l.holder, connectorLockTTL, 0)
- if lock != nil {
- acquired, err := lock.Acquire(context.Background())
- if err != nil {
- common.Warn("connector lock acquire transport error",
- zap.String("key", key), zap.Error(err))
- }
- if !acquired {
- l.mu.Lock()
- delete(l.local, key)
- l.mu.Unlock()
- return ConnectorLockLease{}, false
- }
- } else {
+ if lock == nil {
+ l.mu.Lock()
+ delete(l.local, key)
+ l.mu.Unlock()
+ return ConnectorLockLease{}, false, fmt.Errorf("create distributed connector lock %q", key)
+ }
+ acquired, err := lock.Acquire(context.Background())
+ if err != nil {
+ l.mu.Lock()
+ delete(l.local, key)
+ l.mu.Unlock()
+ return ConnectorLockLease{}, false, fmt.Errorf("acquire distributed connector lock %q: %w", key, err)
+ }
+ if !acquired {
l.mu.Lock()
delete(l.local, key)
l.mu.Unlock()
- return ConnectorLockLease{}, false
+ return ConnectorLockLease{}, false, nil
}
l.mu.Lock()
l.redis[key] = lock
l.mu.Unlock()
- return newConnectorLockLease(), true
+ return newConnectorLockLease(), true, nil
}
- return newConnectorLockLease(), true
+ return newConnectorLockLease(), true, nil
}
// Unlock releases the connector/KB lock.
diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go
index fba51d3f50..184d1ad899 100644
--- a/internal/syncer/syncer_test.go
+++ b/internal/syncer/syncer_test.go
@@ -199,14 +199,31 @@ type closingLocker struct {
db *gorm.DB
}
-func (l closingLocker) TryLock(connectorID, kbID string) (ConnectorLockLease, bool) {
+func (l closingLocker) TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error) {
sqlDB, _ := l.db.DB()
_ = sqlDB.Close()
- return ConnectorLockLease{}, false
+ return ConnectorLockLease{}, false, nil
}
func (l closingLocker) Unlock(connectorID, kbID string) {}
+type cancelingErrorLocker struct {
+ cancel context.CancelFunc
+ beforeError func()
+}
+
+func (l cancelingErrorLocker) TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error) {
+ if l.cancel != nil {
+ l.cancel()
+ }
+ if l.beforeError != nil {
+ l.beforeError()
+ }
+ return ConnectorLockLease{}, false, errors.New("lock backend unavailable")
+}
+
+func (l cancelingErrorLocker) Unlock(connectorID, kbID string) {}
+
// DeleteDocument records one delete.
func (d *fakeDeleter) DeleteDocument(ctx context.Context, docID string) error {
d.mu.Lock()
@@ -706,8 +723,9 @@ func TestNATSTaskWorkerClaimsAndAcksOnSuccess(t *testing.T) {
}
}
-// TestNATSTaskWorkerAcksUnclaimableMessage verifies duplicate/stale messages do not redeliver forever.
-func TestNATSTaskWorkerAcksUnclaimableMessage(t *testing.T) {
+// TestNATSTaskWorkerNacksRunningUnclaimableMessage keeps a delivery pending
+// until the active owner completes or restores the task to schedule state.
+func TestNATSTaskWorkerNacksRunningUnclaimableMessage(t *testing.T) {
db := setupSyncerDB(t)
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error; err != nil {
@@ -718,6 +736,20 @@ func TestNATSTaskWorkerAcksUnclaimableMessage(t *testing.T) {
handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-1", TaskType: common.TaskTypeSyncer}}
worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinator(taskDAO, taskService, newTestRegistry(nil), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())
worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1", Handle: handle})
+ if handle.acks != 0 || handle.nacks != 1 {
+ t.Fatalf("settlement acks=%d nacks=%d", handle.acks, handle.nacks)
+ }
+}
+
+func TestNATSTaskWorkerAcksMissingTaskMessage(t *testing.T) {
+ db := setupSyncerDB(t)
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "missing-task", TaskType: common.TaskTypeSyncer}}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, nil, nil)
+
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "missing-task", Handle: handle})
+
if handle.acks != 1 || handle.nacks != 0 {
t.Fatalf("settlement acks=%d nacks=%d", handle.acks, handle.nacks)
}
@@ -757,6 +789,75 @@ func TestNATSTaskWorkerRetriesStillScheduledUnclaimedTask(t *testing.T) {
}
}
+// TestNATSTaskWorkerPublishesLockErrorRetryAfterCancellation verifies that a
+// restored task is still published when worker cancellation races with a lock error.
+func TestNATSTaskWorkerPublishesLockErrorRetryAfterCancellation(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ published := make(chan string, 1)
+ broker := &fakeSyncTaskBroker{onPublish: func(taskID string) { published <- taskID }}
+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskDAO, broker)
+ t.Cleanup(scheduler.stopTimers)
+ ctx, cancel := context.WithCancel(t.Context())
+ failedReschedule := false
+ worker := NewTaskWorker(
+ make(chan TaskEnvelope, 1),
+ taskDAO,
+ taskService,
+ nil,
+ cancelingErrorLocker{
+ cancel: cancel,
+ beforeError: func() {
+ if err := db.Callback().Update().Before("gorm:update").Register("test:fail_first_reschedule", func(tx *gorm.DB) {
+ if !failedReschedule && tx.Statement.Table == "sync_logs" {
+ failedReschedule = true
+ tx.AddError(errors.New("temporary reschedule failure"))
+ }
+ }); err != nil {
+ t.Fatalf("register reschedule failure callback: %v", err)
+ }
+ },
+ },
+ ).WithScheduler(scheduler)
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-1", TaskType: common.TaskTypeSyncer}}
+
+ worker.handle(ctx, TaskEnvelope{TaskID: "task-1", Handle: handle})
+
+ if handle.acks != 1 || handle.nacks != 0 {
+ t.Fatalf("settlement acks=%d nacks=%d, want ack without nack", handle.acks, handle.nacks)
+ }
+ if !failedReschedule {
+ t.Fatal("reschedule failure callback was not exercised")
+ }
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load restored task: %v", err)
+ }
+ if task.Status != dao.SyncStatusSchedule {
+ t.Fatalf("task status = %s, want schedule", task.Status)
+ }
+ scheduler.timerMu.Lock()
+ timer := scheduler.timers["task-1"]
+ if timer != nil {
+ timer.Reset(time.Millisecond)
+ }
+ scheduler.timerMu.Unlock()
+ if timer == nil {
+ t.Fatal("lock-error retry timer was not registered")
+ }
+
+ select {
+ case taskID := <-published:
+ if taskID != "task-1" {
+ t.Fatalf("published task ID = %q, want task-1", taskID)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("lock-error retry was not published after worker cancellation")
+ }
+}
+
func TestNATSTaskWorkerNacksWhenLockContentionRescheduleFails(t *testing.T) {
db := setupSyncerDB(t)
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.go
index f2cb6667db..fe1c572c35 100644
--- a/internal/syncer/task_worker.go
+++ b/internal/syncer/task_worker.go
@@ -38,6 +38,11 @@ type TaskWorker struct {
scheduler *Scheduler
}
+const (
+ rescheduleClaimedAttempts = 3
+ rescheduleClaimedRetryDelay = 50 * time.Millisecond
+)
+
// NewTaskWorker creates a bounded task worker pool.
func NewTaskWorker(queue <-chan TaskEnvelope, taskDAO *dao.SyncTaskDAO, taskService *service.SyncTaskService, coordinator *TaskCoordinator, locker ConnectorLocker) *TaskWorker {
return &TaskWorker{queue: queue, taskDAO: taskDAO, taskService: taskService, coordinator: coordinator, locker: locker}
@@ -91,8 +96,20 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
return
}
if !claimed {
- _ = envelope.Handle.Ack() // this task has been claimed by other worker
- w.scheduleRetryIfTaskScheduled(ctx, envelope.TaskID, 3*time.Second)
+ status, found, statusErr := w.taskDAO.GetTaskStatus(ctx, envelope.TaskID)
+ if statusErr != nil {
+ common.Warn("syncer unclaimed task status lookup failed", zap.String("task_id", envelope.TaskID), zap.Error(statusErr))
+ _ = envelope.Handle.Nack()
+ return
+ }
+ if status == dao.SyncStatusRunning {
+ _ = envelope.Handle.Nack()
+ return
+ }
+ _ = envelope.Handle.Ack()
+ if found && status == dao.SyncStatusSchedule {
+ w.scheduleRetry(ctx, envelope.TaskID, 3*time.Second)
+ }
return
}
}
@@ -101,14 +118,14 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
taskContext, err := w.taskDAO.GetTaskContext(ctx, envelope.TaskID)
if err != nil {
if ctx.Err() != nil { // exiting
- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID); err != nil {
+ if _, err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID, nil); err != nil {
common.Warn("syncer task reschedule failed after context cancellation", zap.String("task_id", envelope.TaskID), zap.Error(err))
}
nackEnvelope(envelope)
return
}
if failErr := w.taskDAO.FailTask(ctx, envelope.TaskID, "", syncTaskErrorMessage(err), 1); failErr != nil { // getContext failed
- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID); err != nil {
+ if _, err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID, nil); err != nil {
common.Warn("syncer task reschedule failed after context load failure", zap.String("task_id", envelope.TaskID), zap.Error(err))
}
nackEnvelope(envelope)
@@ -119,14 +136,31 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
}
// lock the connector and the KB
- lease, locked := w.locker.TryLock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)
+ lease, locked, lockErr := w.locker.TryLock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)
+ if lockErr != nil {
+ common.Warn("syncer connector lock acquisition failed", zap.String("task_id", taskContext.Task.ID), zap.Error(lockErr))
+ restored, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted)
+ if rescheduleErr != nil {
+ common.Warn("syncer task reschedule failed after lock error", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))
+ nackEnvelope(envelope)
+ return
+ }
+ if restored {
+ w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)
+ }
+ ackEnvelope(envelope)
+ return
+ }
if !locked {
- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {
- common.Warn("syncer task reschedule failed after lock contention", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
+ restored, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted)
+ if rescheduleErr != nil {
+ common.Warn("syncer task reschedule failed after lock contention", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))
nackEnvelope(envelope)
return
}
- w.scheduleRetry(ctx, taskContext.Task.ID, 3*time.Second)
+ if restored {
+ w.scheduleRetry(ctx, taskContext.Task.ID, 3*time.Second)
+ }
ackEnvelope(envelope)
return
}
@@ -141,20 +175,23 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
return
}
if ctx.Err() != nil { // the task is canceled by system, this need to rerun
- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {
- common.Warn("syncer task reschedule failed after execution cancellation", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
+ restored, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted)
+ if rescheduleErr != nil {
+ common.Warn("syncer task reschedule failed after execution cancellation", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))
nackEnvelope(envelope)
return
}
- w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)
+ if restored {
+ w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)
+ }
ackEnvelope(envelope)
return
}
maxRetries := maxTaskRetries(err)
attempts, failed, transientErr := w.taskDAO.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(err), taskErrorClass(err), maxRetries)
if transientErr != nil {
- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {
- common.Warn("syncer task reschedule failed after failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
+ if _, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted); rescheduleErr != nil {
+ common.Warn("syncer task reschedule failed after failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))
}
nackEnvelope(envelope)
return
@@ -195,26 +232,27 @@ func (w *TaskWorker) scheduleRetry(ctx context.Context, taskID string, delay tim
}
}
-func (w *TaskWorker) scheduleRetryIfTaskScheduled(ctx context.Context, taskID string, delay time.Duration) {
- if w.scheduler == nil || taskID == "" {
- return
- }
- taskContext, err := w.taskDAO.GetTaskContext(ctx, taskID)
- if err != nil {
- common.Warn("syncer retry task lookup failed", zap.String("task_id", taskID), zap.Error(err))
- return
- }
- if taskContext.Task.Status != dao.SyncStatusSchedule {
- return
- }
- w.scheduleRetry(ctx, taskID, delay)
-}
-
-func (w *TaskWorker) rescheduleClaimed(ctx context.Context, taskID string) error {
+func (w *TaskWorker) rescheduleClaimed(ctx context.Context, taskID string, startedAt *time.Time) (bool, error) {
if w == nil || w.taskDAO == nil || taskID == "" {
- return nil
+ return false, nil
+ }
+ var err error
+ for attempt := 0; attempt < rescheduleClaimedAttempts; attempt++ {
+ var restored bool
+ restored, err = w.taskDAO.RescheduleClaimed(ctx, taskID, startedAt)
+ if err == nil {
+ return restored, nil
+ }
+ if attempt == rescheduleClaimedAttempts-1 {
+ break
+ }
+ select {
+ case <-ctx.Done():
+ return false, errors.Join(err, ctx.Err())
+ case <-time.After(rescheduleClaimedRetryDelay):
+ }
}
- return w.taskDAO.RescheduleClaimed(ctx, taskID)
+ return false, err
}
func syncTaskErrorMessage(err error) string {