RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(kvrocks): propagate SetNX errors (#20746)
eleven914e2d1c25e19e1
diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.goindex dac007038a..8c946ae9dc 100644--- a/internal/dao/sync_task.go+++ b/internal/dao/sync_task.go@@ -278,28 +278,60 @@ func (d *SyncTaskDAO) IsTaskCanceled(ctx context.Context, taskID string) (bool,return task.Status == SyncStatusCancel, nil}+// GetTaskStatus returns the current status of a sync task. found is false when+// the task no longer exists, allowing consumers to acknowledge stale messages.+func (d *SyncTaskDAO) GetTaskStatus(ctx context.Context, taskID string) (status string, found bool, err error) {+ var task entity.SyncLogs+ err = d.db.WithContext(ctx).Select("status").Where("id = ?", taskID).Take(&task).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return "", false, nil+ }+ if err != nil {+ return "", false, err+ }+ return task.Status, true, nil+}+// MarkConnectorRunning marks a connector running.func (d *SyncTaskDAO) MarkConnectorRunning(ctx context.Context, connectorID string) error {return d.db.WithContext(ctx).Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusRunning).Error}-// RescheduleClaimed puts a claimed task back into schedule state.-func (d *SyncTaskDAO) RescheduleClaimed(ctx context.Context, taskID string) error {- return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+// RescheduleClaimed puts a claimed task back into schedule state. When+// startedAt is non-nil, only the worker that owns that exact claim may restore+// it, so a delayed retry cannot overwrite a newer claim.+func (d *SyncTaskDAO) RescheduleClaimed(ctx context.Context, taskID string, startedAt *time.Time) (bool, error) {+ var restored bool+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {var task entity.SyncLogs- if err := tx.WithContext(ctx).Where("id = ? AND status = ?", taskID, SyncStatusRunning).First(&task).Error; err != nil {+ query := tx.WithContext(ctx).Where("id = ? AND status = ?", taskID, SyncStatusRunning)+ if startedAt != nil {+ query = query.Where("time_started = ?", *startedAt)+ }+ if err := query.First(&task).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {return nil}return err}- if err := tx.Model(&entity.SyncLogs{}).- Where("id = ? AND status = ?", taskID, SyncStatusRunning).- Update("status", SyncStatusSchedule).Error; err != nil {+ update := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", taskID, SyncStatusRunning)+ if startedAt != nil {+ update = update.Where("time_started = ?", *startedAt)+ }+ result := update.Update("status", SyncStatusSchedule)+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected == 0 {+ return nil+ }+ if err := tx.Model(&entity.Connector{}).Where("id = ? AND status = ?", task.ConnectorID, SyncStatusRunning).Update("status", SyncStatusSchedule).Error; err != nil {return err}- return tx.Model(&entity.Connector{}).Where("id = ? AND status = ?", task.ConnectorID, SyncStatusRunning).Update("status", SyncStatusSchedule).Error+ restored = true+ return nil})+ return restored, err}// FailTask marks a task failed without advancing its poll waterline.diff --git a/internal/dao/sync_task_test.go b/internal/dao/sync_task_test.goindex 28a6f99383..aa9052f0ca 100644--- a/internal/dao/sync_task_test.go+++ b/internal/dao/sync_task_test.go@@ -21,6 +21,7 @@ import ("fmt""strings""testing"+ "time""github.com/glebarez/sqlite""gorm.io/gorm"@@ -113,6 +114,50 @@ func insertRunningSyncTask(t *testing.T, db *gorm.DB, taskID string, errorCount,}}+func TestRescheduleClaimedPreservesNewerClaim(t *testing.T) {+ db := setupSyncTaskTestDB(t)+ if err := db.AutoMigrate(&entity.Connector{}); err != nil {+ t.Fatalf("migrate connector: %v", err)+ }+ dao := NewSyncTaskDAO(db)+ staleStartedAt := time.Now().Truncate(time.Millisecond)+ currentStartedAt := staleStartedAt.Add(time.Second)+ if err := db.Create(&entity.SyncLogs{+ ID: "task-1",+ ConnectorID: "conn-1",+ KbID: "kb-1",+ TaskType: TaskTypeSync,+ Status: SyncStatusRunning,+ TimeStarted: ¤tStartedAt,+ }).Error; err != nil {+ t.Fatalf("insert task: %v", err)+ }++ restored, err := dao.RescheduleClaimed(t.Context(), "task-1", &staleStartedAt)+ if err != nil {+ t.Fatalf("reschedule stale claim: %v", err)+ }+ if restored {+ t.Fatal("stale claim restored a newer claim")+ }++ var task entity.SyncLogs+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != SyncStatusRunning {+ t.Fatalf("status = %s, want running", task.Status)+ }++ restored, err = dao.RescheduleClaimed(t.Context(), "task-1", ¤tStartedAt)+ if err != nil {+ t.Fatalf("reschedule current claim: %v", err)+ }+ if !restored {+ t.Fatal("current claim was not restored")+ }+}+// TestHandleTransientFailureUsesClassScopedRetryBudget verifies the retry// comparison uses a counter scoped to the current error class: failures under// one class must not consume another class's budget, and ErrorCount keeps thediff --git a/internal/engine/kvrocks/redis.go b/internal/engine/kvrocks/redis.goindex 146bb0350d..6b2a006800 100644--- a/internal/engine/kvrocks/redis.go+++ b/internal/engine/kvrocks/redis.go@@ -368,32 +368,24 @@ func (r *Client) Set(ctx context.Context, key string, value string, exp time.Durreturn true}-// SetNX sets value only if key does not exist-//-// Deprecated: prefer SetNXErr, which surfaces transport errors instead of-// collapsing them into "lock held". Callers that only need a yes/no answer-// for an idempotent write can keep using SetNX; callers that retry on-// contention (notably DistributedLock.Acquire / SpinAcquire) should call-// SetNXErr so a connection outage is not mistaken for a held lock.-func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) bool {- ok, _ := r.SetNXErr(ctx, key, value, exp)- return ok-}--// SetNXErr sets value only if key does not exist and reports whether the-// Redis/Kvrocks backend could be reached. ok=false with err=nil means the-// key already existed (lock contested); ok=false with err!=nil means the-// transport failed (caller should fail fast, not retry on contention).-func (r *Client) SetNXErr(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {- if r.client == nil {+// SetNX sets value only if key does not exist.+// It returns false with a nil error when the key already exists.+func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {+ if r == nil || r.client == nil {+ return false, errors.New("kvrocks client is not initialized")+ }+ result, err := r.client.SetArgs(ctx, key, value, redis.SetArgs{+ Mode: "NX",+ TTL: exp,+ }).Result()+ if errors.Is(err, redis.Nil) {return false, nil}- ok, err := r.client.SetNX(ctx, key, value, exp).Result()if err != nil {common.Warn("Redis SetNX error", zap.String("key", key), zap.Error(err))- return false, err+ return false, fmt.Errorf("set key %q if absent: %w", key, err)}- return ok, nil+ return result == "OK", nil}// GetOrCreateKey atomically retrieves an existing key or creates a new one@@ -835,17 +827,20 @@ func (r *Client) QueueInfo(ctx context.Context, queue, groupName string) (map[streturn nil, nil}-// DeleteIfEqual deletes key if its value equals expected value (atomic)-func (r *Client) DeleteIfEqual(ctx context.Context, key, expectedValue string) bool {- if r.client == nil {- return false+// deleteIfEqual deletes key if its value equals expected value atomically.+func (r *Client) deleteIfEqual(ctx context.Context, key, expectedValue string) (bool, error) {+ if r == nil || r.client == nil {+ return false, errors.New("kvrocks client is not initialized")+ }+ if r.luaDeleteIfEqual == nil {+ return false, errors.New("kvrocks delete-if-equal script is not initialized")}- result, err := r.luaDeleteIfEqual.Run(ctx, r.client, []string{key}, expectedValue).Result()+ result, err := r.luaDeleteIfEqual.Run(ctx, r.client, []string{key}, expectedValue).Int64()if err != nil {- common.Warn("Redis DeleteIfEqual error", zap.Error(err))- return false+ common.Warn("Redis DeleteIfEqual error", zap.String("key", key), zap.Error(err))+ return false, fmt.Errorf("delete key %q if value matches: %w", key, err)}- return result.(int64) == 1+ return result == 1, nil}// Delete deletes a key@@ -911,20 +906,17 @@ func NewDistributedLock(lockKey string, lockValue string, timeout time.Duration,}}-// Acquire attempts to take the lock. Returns (true, nil) on success,-// (false, nil) when the lock is held by someone else, and (false, err)-// when the underlying SETNX itself could not be issued (transport / auth).-// The third case is what used to be collapsed into "lock held" by the-// bool-only SetNX path, which made every DocumentService.WithDocument--// MetadataLock caller spin for its full 15 s budget during a Redis/-// Kvrocks outage instead of failing immediately.+// Acquire acquires the lock. It returns false with a nil error when another+// owner holds the lock.func (l *DistributedLock) Acquire(ctx context.Context) (bool, error) {- if l.client == nil {- return false, nil+ if l == nil || l.client == nil {+ return false, errors.New("distributed lock client is not initialized")+ }+ // Remove a lock left by this owner before trying to acquire it again.+ if _, err := l.client.deleteIfEqual(ctx, l.lockKey, l.lockValue); err != nil {+ return false, err}- // Delete if stale- l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)- return l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)+ return l.client.SetNX(ctx, l.lockKey, l.lockValue, l.timeout)}// SpinAcquire keeps trying to acquire the lock. Returns the underlying@@ -937,12 +929,11 @@ func (l *DistributedLock) SpinAcquire(ctx context.Context) error {case <-ctx.Done():return ctx.Err()default:- l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)- ok, err := l.client.SetNXErr(ctx, l.lockKey, l.lockValue, l.timeout)+ acquired, err := l.Acquire(ctx)if err != nil {return err}- if ok {+ if acquired {return nil}time.Sleep(10 * time.Second)@@ -952,10 +943,11 @@ func (l *DistributedLock) SpinAcquire(ctx context.Context) error {// Release releases the lockfunc (l *DistributedLock) Release(ctx context.Context) bool {- if l.client == nil {+ if l == nil || l.client == nil {return false}- return l.client.DeleteIfEqual(ctx, l.lockKey, l.lockValue)+ released, err := l.client.deleteIfEqual(ctx, l.lockKey, l.lockValue)+ return err == nil && released}// TokenBucket token bucket rate limiterdiff --git a/internal/engine/kvrocks/redis_test.go b/internal/engine/kvrocks/redis_test.goindex e38cd6dcb1..1e638b1876 100644--- a/internal/engine/kvrocks/redis_test.go+++ b/internal/engine/kvrocks/redis_test.go@@ -18,7 +18,6 @@ package kvrocksimport ("context"- "errors""testing""time"@@ -146,127 +145,96 @@ func TestEvalTokenBucketStrict_NilClient(t *testing.T) {}}-// TestSetNXErr_AllowedWhenNew pins the happy path: SetNXErr reports-// (true, nil) on a brand-new key. Regression guard for issue #20660.-func TestSetNXErr_AllowedWhenNew(t *testing.T) {+func TestClientSetNXDistinguishesContention(t *testing.T) {r, _ := newStrictTestClient(t)ctx := t.Context()- ok, err := r.SetNXErr(ctx, "lock:test:new", "v", time.Minute)+ acquired, err := r.SetNX(ctx, "lock:document", "owner-1", time.Minute)if err != nil {- t.Fatalf("unexpected error: %v", err)+ t.Fatalf("first SetNX returned error: %v", err)}- if !ok {- t.Fatalf("expected ok=true on a fresh key")+ if !acquired {+ t.Fatal("first SetNX acquired = false, want true")}-}--// TestSetNXErr_FalseOnContention confirms the second concurrent caller-// sees (false, nil), not (false, err). Only the transport-failure case-// should return an error - keep the two outcomes distinguishable so the-// DistributedLock wait-for-lock loop can retry on the former and fail-// fast on the latter.-func TestSetNXErr_FalseOnContention(t *testing.T) {- r, _ := newStrictTestClient(t)- ctx := t.Context()- if ok, err := r.SetNXErr(ctx, "lock:test:contended", "first", time.Minute); err != nil || !ok {- t.Fatalf("seed call: ok=%v err=%v", ok, err)- }- ok, err := r.SetNXErr(ctx, "lock:test:contended", "second", time.Minute)+ acquired, err = r.SetNX(ctx, "lock:document", "owner-2", time.Minute)if err != nil {- t.Fatalf("contention should NOT surface a transport error, got %v", err)+ t.Fatalf("contended SetNX returned error: %v", err)}- if ok {- t.Fatalf("expected ok=false on a held key")+ if acquired {+ t.Fatal("contended SetNX acquired = true, want false")}}-// TestSetNXErr_TransportErrorFailsFast is the core regression for-// #20660. With miniredis closed, the underlying SETNX cannot reach the-// server; SetNXErr must surface that as a non-nil error instead of-// returning false and forcing every distributed-lock caller to spin for-// its full timeout budget. Mirrors the EvalTokenBucketStrict-// fail-closed contract in the same package.-func TestSetNXErr_TransportErrorFailsFast(t *testing.T) {+func TestClientSetNXReturnsTransportError(t *testing.T) {r, mr := newStrictTestClient(t)mr.Close()ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)defer cancel()- ok, err := r.SetNXErr(ctx, "lock:test:down", "v", time.Minute)+ acquired, err := r.SetNX(ctx, "lock:document", "owner-1", time.Minute)if err == nil {- t.Fatalf("expected transport error, got nil (ok=%v)", ok)+ t.Fatal("SetNX error = nil, want transport error")}- if ok {- t.Fatalf("expected ok=false on transport failure (fail-closed)")+ if acquired {+ t.Fatal("SetNX acquired = true on transport failure, want false")}}-// TestDistributedLock_AcquireTransportError mirrors the issue body:-// DistributedLock.Acquire must surface the underlying transport error-// rather than collapsing it into (false, nil), otherwise every polling-// caller (the two withDDLLock waitForLock loops and any future spin-// caller) will treat "Redis is down" as "lock held by someone else".-func TestDistributedLock_AcquireTransportError(t *testing.T) {- r, mr := newStrictTestClient(t)- mr.Close()- // Bypass NewDistributedLock's globalClient guard so we can drive- // Acquire against a known-down client.- lock := &DistributedLock{- client: r,- lockKey: "lock:test:down",- lockValue: "v",- timeout: time.Minute,- blockingTimeout: 0,- }-- ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)- defer cancel()+func TestClientSetNXReturnsErrorForNilClient(t *testing.T) {+ var r *Client- acquired, err := lock.Acquire(ctx)+ acquired, err := r.SetNX(t.Context(), "lock:document", "owner-1", time.Minute)if err == nil {- t.Fatalf("expected transport error from Acquire, got nil (acquired=%v)", acquired)+ t.Fatal("SetNX error = nil, want initialization error")}if acquired {- t.Fatalf("expected acquired=false on transport failure (fail-closed)")+ t.Fatal("SetNX acquired = true for nil client, want false")}}-// TestDistributedLock_SpinAcquireSurfacesTransportError confirms the-// second regression the issue called out: SpinAcquire used to swallow-// the underlying SetNX error and only return ctx.Err() at the next-// select-tick. With Redis down, SpinAcquire must return the transport-// error directly so the metadata-write path can fail fast (target:-// ~3s on a real outage, the go-redis connection-pool exhaustion time,-// rather than the 10s internal sleep plus caller budget).-//-// The 5s ctx is sized to give go-redis's 5-attempt connection pool-// exhaustion time to surface the connection-refused error before the-// test deadline cancels the call - otherwise the assertion would-// conflate "transport failure" with "ctx timed out first" (because-// go-redis's .Result() returns ctx.Err() when the parent context-// deadline beats the dial to the failed endpoint).-func TestDistributedLock_SpinAcquireSurfacesTransportError(t *testing.T) {+func TestDistributedLockSpinAcquireReturnsTransportError(t *testing.T) {r, mr := newStrictTestClient(t)mr.Close()lock := &DistributedLock{- client: r,- lockKey: "lock:test:down",- lockValue: "v",- timeout: time.Minute,- blockingTimeout: 0,+ client: r,+ lockKey: "lock:document",+ lockValue: "owner-1",+ timeout: time.Minute,}- ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)+ ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)defer cancel()- err := lock.SpinAcquire(ctx)+ if err := lock.SpinAcquire(ctx); err == nil {+ t.Fatal("SpinAcquire error = nil, want transport error")+ }+}++func TestDistributedLockAcquireReturnsCleanupError(t *testing.T) {+ r, _ := newStrictTestClient(t)+ r.luaDeleteIfEqual = redis.NewScript(`return redis.error_reply("cleanup failed")`)+ lock := &DistributedLock{+ client: r,+ lockKey: "lock:document",+ lockValue: "owner-1",+ timeout: time.Minute,+ }++ acquired, err := lock.Acquire(t.Context())if err == nil {- t.Fatalf("expected transport error from SpinAcquire, got nil")+ t.Fatal("Acquire error = nil, want cleanup error")+ }+ if acquired {+ t.Fatal("Acquire acquired = true on cleanup failure, want false")+ }++ exists, err := r.client.Exists(t.Context(), lock.lockKey).Result()+ if err != nil {+ t.Fatalf("check lock key after cleanup failure: %v", err)}- if errors.Is(err, context.DeadlineExceeded) {- t.Fatalf("SpinAcquire returned ctx.Err() (deadline) instead of the transport error: %v", err)+ if exists != 0 {+ t.Fatal("Acquire attempted SetNX after cleanup failure")}}diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.goindex 8f40d4bb5e..59e4536368 100644--- a/internal/engine/oceanbase/schema.go+++ b/internal/engine/oceanbase/schema.go@@ -326,38 +326,39 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()}timeout := time.Duration(timeoutSeconds) * time.Seconddistributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)+ acquired := falseif distributed != nil {- acquired, err := distributed.Acquire(ctx)+ acquired, err = distributed.Acquire(ctx)if err != nil {- return err+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)}- if !acquired {- deadline := time.NewTimer(timeout)- defer deadline.Stop()- ticker := time.NewTicker(time.Second)- defer ticker.Stop()- waitForLock:- for {- select {- case <-ctx.Done():- return ctx.Err()- case <-deadline.C:- return fmt.Errorf("timeout waiting for DDL %s", lockName)- case <-ticker.C:- exists, err = check()- if err != nil {- return err- }- if exists {- return nil- }- got, err := distributed.Acquire(ctx)- if err != nil {- return err- }- if got {- break waitForLock- }+ }+ if distributed != nil && !acquired {+ deadline := time.NewTimer(timeout)+ defer deadline.Stop()+ ticker := time.NewTicker(time.Second)+ defer ticker.Stop()+ waitForLock:+ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-deadline.C:+ return fmt.Errorf("timeout waiting for DDL %s", lockName)+ case <-ticker.C:+ exists, err = check()+ if err != nil {+ return err+ }+ if exists {+ return nil+ }+ acquired, err = distributed.Acquire(ctx)+ if err != nil {+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)+ }+ if acquired {+ break waitForLock}}}diff --git a/internal/engine/vastbase/schema.go b/internal/engine/vastbase/schema.goindex 1b5cf1678a..f3f1fc71a8 100644--- a/internal/engine/vastbase/schema.go+++ b/internal/engine/vastbase/schema.go@@ -427,38 +427,39 @@ func (e *Engine) withDDLLock(ctx context.Context, lockName string, check func()timeout := 60 * time.Seconddistributed := kvrocks.NewDistributedLock(lockName, "", timeout, timeout)+ acquired := falseif distributed != nil {- acquired, err := distributed.Acquire(ctx)+ acquired, err = distributed.Acquire(ctx)if err != nil {- return err+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)}- if !acquired {- deadline := time.NewTimer(timeout)- defer deadline.Stop()- ticker := time.NewTicker(time.Second)- defer ticker.Stop()- waitForLock:- for {- select {- case <-ctx.Done():- return ctx.Err()- case <-deadline.C:- return fmt.Errorf("timeout waiting for DDL %s", lockName)- case <-ticker.C:- exists, err = check()- if err != nil {- return err- }- if exists {- return nil- }- got, err := distributed.Acquire(ctx)- if err != nil {- return err- }- if got {- break waitForLock- }+ }+ if distributed != nil && !acquired {+ deadline := time.NewTimer(timeout)+ defer deadline.Stop()+ ticker := time.NewTicker(time.Second)+ defer ticker.Stop()+ waitForLock:+ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-deadline.C:+ return fmt.Errorf("timeout waiting for DDL %s", lockName)+ case <-ticker.C:+ exists, err = check()+ if err != nil {+ return err+ }+ if exists {+ return nil+ }+ acquired, err = distributed.Acquire(ctx)+ if err != nil {+ return fmt.Errorf("acquire distributed DDL lock %s: %w", lockName, err)+ }+ if acquired {+ break waitForLock}}}diff --git a/internal/server/variable.go b/internal/server/variable.goindex 693cb53eaf..bfa11f6079 100644--- a/internal/server/variable.go+++ b/internal/server/variable.go@@ -36,7 +36,7 @@ type Variables struct {type VariableStore interface {Get(ctx context.Context, key string) (string, error)Set(ctx context.Context, key string, value string, exp time.Duration) bool- SetNX(ctx context.Context, key string, value string, exp time.Duration) bool+ SetNX(ctx context.Context, key string, value string, exp time.Duration) (bool, error)}var (@@ -144,7 +144,11 @@ func GetOrCreateKey(ctx context.Context, store VariableStore, key string, newValcommon.Info("Generating new value for key", zap.String("key", key))// Try to set with NX (only if not exists) - ensures atomicity- if store.SetNX(ctx, key, newValue, SecretKeyTTL) {+ created, err := store.SetNX(ctx, key, newValue, SecretKeyTTL)+ if err != nil {+ return "", fmt.Errorf("failed to create key %q: %w", key, err)+ }+ if created {common.Info("New value stored successfully", zap.String("key", key))return newValue, nil}diff --git a/internal/server/variable_test.go b/internal/server/variable_test.gonew file mode 100644index 0000000000..842ac1c3a5--- /dev/null+++ b/internal/server/variable_test.go@@ -0,0 +1,94 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package server++import (+ "context"+ "errors"+ "strings"+ "testing"+ "time"+)++type variableStoreStub struct {+ value string+ getErr error+ setNX bool+ setNXErr error+}++func (s *variableStoreStub) Get(context.Context, string) (string, error) {+ return s.value, s.getErr+}++func (s *variableStoreStub) Set(context.Context, string, string, time.Duration) bool {+ return true+}++func (s *variableStoreStub) SetNX(context.Context, string, string, time.Duration) (bool, error) {+ return s.setNX, s.setNXErr+}++func TestGetOrCreateKeyReturnsSetNXError(t *testing.T) {+ transportErr := errors.New("kvrocks unavailable")+ store := &variableStoreStub{setNXErr: transportErr}++ value, err := GetOrCreateKey(t.Context(), store, "ragflow:key", "generated")+ if err == nil {+ t.Fatal("GetOrCreateKey error = nil, want SetNX error")+ }+ if !errors.Is(err, transportErr) {+ t.Fatalf("GetOrCreateKey error = %v, want wrapped transport error", err)+ }+ if value != "" {+ t.Fatalf("GetOrCreateKey value = %q, want empty string", value)+ }+ if !strings.Contains(err.Error(), "ragflow:key") {+ t.Fatalf("GetOrCreateKey error = %q, want key context", err)+ }+}++func TestGetOrCreateKeyHandlesSetNXContention(t *testing.T) {+ // The first read misses. After SetNX reports contention, the second read+ // observes the value created by the competing process.+ storeWithSequence := &sequenceVariableStore{values: []string{"", "competing"}}+ value, err := GetOrCreateKey(t.Context(), storeWithSequence, "ragflow:key", "generated")+ if err != nil {+ t.Fatalf("GetOrCreateKey returned error: %v", err)+ }+ if value != "competing" {+ t.Fatalf("GetOrCreateKey value = %q, want %q", value, "competing")+ }+}++type sequenceVariableStore struct {+ values []string+}++func (s *sequenceVariableStore) Get(context.Context, string) (string, error) {+ value := s.values[0]+ s.values = s.values[1:]+ return value, nil+}++func (s *sequenceVariableStore) Set(context.Context, string, string, time.Duration) bool {+ return true+}++func (s *sequenceVariableStore) SetNX(context.Context, string, string, time.Duration) (bool, error) {+ return false, nil+}diff --git a/internal/syncer/connector_lock.go b/internal/syncer/connector_lock.goindex 6e06cf256c..79770d9d2f 100644--- a/internal/syncer/connector_lock.go+++ b/internal/syncer/connector_lock.go@@ -23,13 +23,11 @@ import ("ragflow/internal/engine/kvrocks""sync""time"-- "go.uber.org/zap")// ConnectorLocker serializes work for one connector and knowledge base.type ConnectorLocker interface {- TryLock(connectorID, kbID string) (ConnectorLockLease, bool)+ TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error)Unlock(connectorID, kbID string)}@@ -54,45 +52,46 @@ func NewConnectorLock() *ConnectorLock {}// TryLock attempts to acquire the connector/KB lock without blocking.-func (l *ConnectorLock) TryLock(connectorID, kbID string) (ConnectorLockLease, bool) {+func (l *ConnectorLock) TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error) {if l == nil {- return ConnectorLockLease{}, false+ return ConnectorLockLease{}, false, nil}key := connectorLockKey(connectorID, kbID)l.mu.Lock()if _, ok := l.local[key]; ok {l.mu.Unlock()- return ConnectorLockLease{}, false+ return ConnectorLockLease{}, false, nil}l.local[key] = struct{}{}l.mu.Unlock()if client := kvrocks.Get(); client != nil {lock := kvrocks.NewDistributedLock(key, l.holder, connectorLockTTL, 0)- if lock != nil {- acquired, err := lock.Acquire(context.Background())- if err != nil {- common.Warn("connector lock acquire transport error",- zap.String("key", key), zap.Error(err))- }- if !acquired {- l.mu.Lock()- delete(l.local, key)- l.mu.Unlock()- return ConnectorLockLease{}, false- }- } else {+ if lock == nil {+ l.mu.Lock()+ delete(l.local, key)+ l.mu.Unlock()+ return ConnectorLockLease{}, false, fmt.Errorf("create distributed connector lock %q", key)+ }+ acquired, err := lock.Acquire(context.Background())+ if err != nil {+ l.mu.Lock()+ delete(l.local, key)+ l.mu.Unlock()+ return ConnectorLockLease{}, false, fmt.Errorf("acquire distributed connector lock %q: %w", key, err)+ }+ if !acquired {l.mu.Lock()delete(l.local, key)l.mu.Unlock()- return ConnectorLockLease{}, false+ return ConnectorLockLease{}, false, nil}l.mu.Lock()l.redis[key] = lockl.mu.Unlock()- return newConnectorLockLease(), true+ return newConnectorLockLease(), true, nil}- return newConnectorLockLease(), true+ return newConnectorLockLease(), true, nil}// Unlock releases the connector/KB lock.diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.goindex fba51d3f50..184d1ad899 100644--- a/internal/syncer/syncer_test.go+++ b/internal/syncer/syncer_test.go@@ -199,14 +199,31 @@ type closingLocker struct {db *gorm.DB}-func (l closingLocker) TryLock(connectorID, kbID string) (ConnectorLockLease, bool) {+func (l closingLocker) TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error) {sqlDB, _ := l.db.DB()_ = sqlDB.Close()- return ConnectorLockLease{}, false+ return ConnectorLockLease{}, false, nil}func (l closingLocker) Unlock(connectorID, kbID string) {}+type cancelingErrorLocker struct {+ cancel context.CancelFunc+ beforeError func()+}++func (l cancelingErrorLocker) TryLock(connectorID, kbID string) (ConnectorLockLease, bool, error) {+ if l.cancel != nil {+ l.cancel()+ }+ if l.beforeError != nil {+ l.beforeError()+ }+ return ConnectorLockLease{}, false, errors.New("lock backend unavailable")+}++func (l cancelingErrorLocker) Unlock(connectorID, kbID string) {}+// DeleteDocument records one delete.func (d *fakeDeleter) DeleteDocument(ctx context.Context, docID string) error {d.mu.Lock()@@ -706,8 +723,9 @@ func TestNATSTaskWorkerClaimsAndAcksOnSuccess(t *testing.T) {}}-// TestNATSTaskWorkerAcksUnclaimableMessage verifies duplicate/stale messages do not redeliver forever.-func TestNATSTaskWorkerAcksUnclaimableMessage(t *testing.T) {+// TestNATSTaskWorkerNacksRunningUnclaimableMessage keeps a delivery pending+// until the active owner completes or restores the task to schedule state.+func TestNATSTaskWorkerNacksRunningUnclaimableMessage(t *testing.T) {db := setupSyncerDB(t)insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error; err != nil {@@ -718,6 +736,20 @@ func TestNATSTaskWorkerAcksUnclaimableMessage(t *testing.T) {handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-1", TaskType: common.TaskTypeSyncer}}worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinator(taskDAO, taskService, newTestRegistry(nil), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1", Handle: handle})+ if handle.acks != 0 || handle.nacks != 1 {+ t.Fatalf("settlement acks=%d nacks=%d", handle.acks, handle.nacks)+ }+}++func TestNATSTaskWorkerAcksMissingTaskMessage(t *testing.T) {+ db := setupSyncerDB(t)+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "missing-task", TaskType: common.TaskTypeSyncer}}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, nil, nil)++ worker.handle(t.Context(), TaskEnvelope{TaskID: "missing-task", Handle: handle})+if handle.acks != 1 || handle.nacks != 0 {t.Fatalf("settlement acks=%d nacks=%d", handle.acks, handle.nacks)}@@ -757,6 +789,75 @@ func TestNATSTaskWorkerRetriesStillScheduledUnclaimedTask(t *testing.T) {}}+// TestNATSTaskWorkerPublishesLockErrorRetryAfterCancellation verifies that a+// restored task is still published when worker cancellation races with a lock error.+func TestNATSTaskWorkerPublishesLockErrorRetryAfterCancellation(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ published := make(chan string, 1)+ broker := &fakeSyncTaskBroker{onPublish: func(taskID string) { published <- taskID }}+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskDAO, broker)+ t.Cleanup(scheduler.stopTimers)+ ctx, cancel := context.WithCancel(t.Context())+ failedReschedule := false+ worker := NewTaskWorker(+ make(chan TaskEnvelope, 1),+ taskDAO,+ taskService,+ nil,+ cancelingErrorLocker{+ cancel: cancel,+ beforeError: func() {+ if err := db.Callback().Update().Before("gorm:update").Register("test:fail_first_reschedule", func(tx *gorm.DB) {+ if !failedReschedule && tx.Statement.Table == "sync_logs" {+ failedReschedule = true+ tx.AddError(errors.New("temporary reschedule failure"))+ }+ }); err != nil {+ t.Fatalf("register reschedule failure callback: %v", err)+ }+ },+ },+ ).WithScheduler(scheduler)+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-1", TaskType: common.TaskTypeSyncer}}++ worker.handle(ctx, TaskEnvelope{TaskID: "task-1", Handle: handle})++ if handle.acks != 1 || handle.nacks != 0 {+ t.Fatalf("settlement acks=%d nacks=%d, want ack without nack", handle.acks, handle.nacks)+ }+ if !failedReschedule {+ t.Fatal("reschedule failure callback was not exercised")+ }+ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load restored task: %v", err)+ }+ if task.Status != dao.SyncStatusSchedule {+ t.Fatalf("task status = %s, want schedule", task.Status)+ }+ scheduler.timerMu.Lock()+ timer := scheduler.timers["task-1"]+ if timer != nil {+ timer.Reset(time.Millisecond)+ }+ scheduler.timerMu.Unlock()+ if timer == nil {+ t.Fatal("lock-error retry timer was not registered")+ }++ select {+ case taskID := <-published:+ if taskID != "task-1" {+ t.Fatalf("published task ID = %q, want task-1", taskID)+ }+ case <-time.After(time.Second):+ t.Fatal("lock-error retry was not published after worker cancellation")+ }+}+func TestNATSTaskWorkerNacksWhenLockContentionRescheduleFails(t *testing.T) {db := setupSyncerDB(t)insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.goindex f2cb6667db..fe1c572c35 100644--- a/internal/syncer/task_worker.go+++ b/internal/syncer/task_worker.go@@ -38,6 +38,11 @@ type TaskWorker struct {scheduler *Scheduler}+const (+ rescheduleClaimedAttempts = 3+ rescheduleClaimedRetryDelay = 50 * time.Millisecond+)+// NewTaskWorker creates a bounded task worker pool.func NewTaskWorker(queue <-chan TaskEnvelope, taskDAO *dao.SyncTaskDAO, taskService *service.SyncTaskService, coordinator *TaskCoordinator, locker ConnectorLocker) *TaskWorker {return &TaskWorker{queue: queue, taskDAO: taskDAO, taskService: taskService, coordinator: coordinator, locker: locker}@@ -91,8 +96,20 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {return}if !claimed {- _ = envelope.Handle.Ack() // this task has been claimed by other worker- w.scheduleRetryIfTaskScheduled(ctx, envelope.TaskID, 3*time.Second)+ status, found, statusErr := w.taskDAO.GetTaskStatus(ctx, envelope.TaskID)+ if statusErr != nil {+ common.Warn("syncer unclaimed task status lookup failed", zap.String("task_id", envelope.TaskID), zap.Error(statusErr))+ _ = envelope.Handle.Nack()+ return+ }+ if status == dao.SyncStatusRunning {+ _ = envelope.Handle.Nack()+ return+ }+ _ = envelope.Handle.Ack()+ if found && status == dao.SyncStatusSchedule {+ w.scheduleRetry(ctx, envelope.TaskID, 3*time.Second)+ }return}}@@ -101,14 +118,14 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {taskContext, err := w.taskDAO.GetTaskContext(ctx, envelope.TaskID)if err != nil {if ctx.Err() != nil { // exiting- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID); err != nil {+ if _, err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID, nil); err != nil {common.Warn("syncer task reschedule failed after context cancellation", zap.String("task_id", envelope.TaskID), zap.Error(err))}nackEnvelope(envelope)return}if failErr := w.taskDAO.FailTask(ctx, envelope.TaskID, "", syncTaskErrorMessage(err), 1); failErr != nil { // getContext failed- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID); err != nil {+ if _, err = w.rescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID, nil); err != nil {common.Warn("syncer task reschedule failed after context load failure", zap.String("task_id", envelope.TaskID), zap.Error(err))}nackEnvelope(envelope)@@ -119,14 +136,31 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {}// lock the connector and the KB- lease, locked := w.locker.TryLock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)+ lease, locked, lockErr := w.locker.TryLock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)+ if lockErr != nil {+ common.Warn("syncer connector lock acquisition failed", zap.String("task_id", taskContext.Task.ID), zap.Error(lockErr))+ restored, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted)+ if rescheduleErr != nil {+ common.Warn("syncer task reschedule failed after lock error", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))+ nackEnvelope(envelope)+ return+ }+ if restored {+ w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)+ }+ ackEnvelope(envelope)+ return+ }if !locked {- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {- common.Warn("syncer task reschedule failed after lock contention", zap.String("task_id", taskContext.Task.ID), zap.Error(err))+ restored, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted)+ if rescheduleErr != nil {+ common.Warn("syncer task reschedule failed after lock contention", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))nackEnvelope(envelope)return}- w.scheduleRetry(ctx, taskContext.Task.ID, 3*time.Second)+ if restored {+ w.scheduleRetry(ctx, taskContext.Task.ID, 3*time.Second)+ }ackEnvelope(envelope)return}@@ -141,20 +175,23 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {return}if ctx.Err() != nil { // the task is canceled by system, this need to rerun- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {- common.Warn("syncer task reschedule failed after execution cancellation", zap.String("task_id", taskContext.Task.ID), zap.Error(err))+ restored, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted)+ if rescheduleErr != nil {+ common.Warn("syncer task reschedule failed after execution cancellation", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))nackEnvelope(envelope)return}- w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)+ if restored {+ w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)+ }ackEnvelope(envelope)return}maxRetries := maxTaskRetries(err)attempts, failed, transientErr := w.taskDAO.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(err), taskErrorClass(err), maxRetries)if transientErr != nil {- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {- common.Warn("syncer task reschedule failed after failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))+ if _, rescheduleErr := w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID, taskContext.Task.TimeStarted); rescheduleErr != nil {+ common.Warn("syncer task reschedule failed after failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(rescheduleErr))}nackEnvelope(envelope)return@@ -195,26 +232,27 @@ func (w *TaskWorker) scheduleRetry(ctx context.Context, taskID string, delay tim}}-func (w *TaskWorker) scheduleRetryIfTaskScheduled(ctx context.Context, taskID string, delay time.Duration) {- if w.scheduler == nil || taskID == "" {- return- }- taskContext, err := w.taskDAO.GetTaskContext(ctx, taskID)- if err != nil {- common.Warn("syncer retry task lookup failed", zap.String("task_id", taskID), zap.Error(err))- return- }- if taskContext.Task.Status != dao.SyncStatusSchedule {- return- }- w.scheduleRetry(ctx, taskID, delay)-}--func (w *TaskWorker) rescheduleClaimed(ctx context.Context, taskID string) error {+func (w *TaskWorker) rescheduleClaimed(ctx context.Context, taskID string, startedAt *time.Time) (bool, error) {if w == nil || w.taskDAO == nil || taskID == "" {- return nil+ return false, nil+ }+ var err error+ for attempt := 0; attempt < rescheduleClaimedAttempts; attempt++ {+ var restored bool+ restored, err = w.taskDAO.RescheduleClaimed(ctx, taskID, startedAt)+ if err == nil {+ return restored, nil+ }+ if attempt == rescheduleClaimedAttempts-1 {+ break+ }+ select {+ case <-ctx.Done():+ return false, errors.Join(err, ctx.Err())+ case <-time.After(rescheduleClaimedRetryDelay):+ }}- return w.taskDAO.RescheduleClaimed(ctx, taskID)+ return false, err}func syncTaskErrorMessage(err error) string {