Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): retry sync task errors with per-class budgets (#19790)

mkaaada97f4c882b03
diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.go
index ae0c7f30e7..8d791a2e7c 100644
--- a/internal/dao/sync_task.go
+++ b/internal/dao/sync_task.go
@@ -294,8 +294,12 @@ func (d *SyncTaskDAO) FailTask(ctx context.Context, taskID, connectorID, message
})
}
-// HandleTransientFailure retries a running task until maxRetries is reached.
-func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connectorID, message string, maxRetries int64) (int64, bool, error) {
+// HandleTransientFailure retries a running task until maxRetries for its error
+// class is reached. errorClass identifies the current failure class; the
+// class-scoped retry counter resets whenever the class changes, so failures of
+// one class never consume another class's retry budget. ErrorCount is kept as
+// the total failure count for diagnostics.
+func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connectorID, message, errorClass string, maxRetries int64) (int64, bool, error) {
var attempts int64
var failed bool
err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
@@ -311,7 +315,11 @@ func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connec
return err
}
- attempts = task.ErrorCount + 1
+ attempts = task.RetryCount + 1
+ if task.ErrorClass != errorClass {
+ // A different failure class starts a fresh retry budget.
+ attempts = 1
+ }
status := SyncStatusSchedule
connectorStatus := SyncStatusSchedule
errorMsg := message
@@ -319,7 +327,7 @@ func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connec
failed = true
status = SyncStatusFail
connectorStatus = SyncStatusFail
- errorMsg = fmt.Sprintf("sync task failed after %d transient retries: %s", maxRetries, message)
+ errorMsg = fmt.Sprintf("sync task failed after %d retries, last error: %s", attempts-1, message)
}
if err := tx.Model(&entity.SyncLogs{}).
@@ -327,7 +335,9 @@ func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connec
Updates(map[string]any{
"status": status,
"error_msg": errorMsg,
- "error_count": attempts,
+ "error_count": task.ErrorCount + 1,
+ "retry_count": attempts,
+ "error_class": errorClass,
}).Error; err != nil {
return err
}
diff --git a/internal/dao/sync_task_test.go b/internal/dao/sync_task_test.go
new file mode 100644
index 0000000000..5d5b607f39
--- /dev/null
+++ b/internal/dao/sync_task_test.go
@@ -0,0 +1,141 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+)
+
+// setupSyncTaskTestDB initializes an in-memory SQLite database for SyncTask DAO tests.
+func setupSyncTaskTestDB(t *testing.T) *gorm.DB {
+ t.Helper()
+
+ dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))
+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
+ TranslateError: true,
+ })
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("failed to get sql DB: %v", err)
+ }
+ sqlDB.SetMaxOpenConns(1)
+
+ if err := db.AutoMigrate(&entity.SyncLogs{}); err != nil {
+ t.Fatalf("failed to migrate SyncLogs: %v", err)
+ }
+ return db
+}
+
+func insertRunningSyncTask(t *testing.T, db *gorm.DB, taskID string, errorCount, retryCount int64, errorClass string) {
+ t.Helper()
+ if err := db.Create(&entity.SyncLogs{
+ ID: taskID,
+ ConnectorID: "conn-1",
+ KbID: "kb-1",
+ TaskType: TaskTypeSync,
+ Status: SyncStatusRunning,
+ ErrorCount: errorCount,
+ RetryCount: retryCount,
+ ErrorClass: errorClass,
+ }).Error; err != nil {
+ t.Fatalf("insert task: %v", err)
+ }
+}
+
+// TestHandleTransientFailureUsesClassScopedRetryBudget verifies the retry
+// comparison uses a counter scoped to the current error class: failures under
+// one class must not consume another class's budget, and ErrorCount keeps the
+// total failure count for diagnostics.
+func TestHandleTransientFailureUsesClassScopedRetryBudget(t *testing.T) {
+ db := setupSyncTaskTestDB(t)
+ dao := NewSyncTaskDAO(db)
+ ctx := context.Background()
+
+ // The task already exhausted the non-transient budget (3 failures), then
+ // hits a transient error whose budget (4) must start fresh.
+ insertRunningSyncTask(t, db, "task-1", 3, 3, "non_transient")
+ attempts, failed, err := dao.HandleTransientFailure(ctx, "task-1", "", "boom", "transient", 4)
+ if err != nil {
+ t.Fatalf("HandleTransientFailure: %v", err)
+ }
+ if failed {
+ t.Fatalf("failed = true, want false (transient budget must reset)")
+ }
+ if attempts != 1 {
+ t.Fatalf("attempts = %d, want 1", attempts)
+ }
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != SyncStatusSchedule {
+ t.Fatalf("status = %s, want schedule", task.Status)
+ }
+ if task.ErrorCount != 4 {
+ t.Fatalf("error_count = %d, want 4 (diagnostics total)", task.ErrorCount)
+ }
+ if task.RetryCount != 1 {
+ t.Fatalf("retry_count = %d, want 1", task.RetryCount)
+ }
+ if task.ErrorClass != "transient" {
+ t.Fatalf("error_class = %q, want transient", task.ErrorClass)
+ }
+
+ // Same class keeps accumulating and fails once the class budget is spent.
+ for i := 2; i <= 4; i++ {
+ // The worker re-claims the scheduled task before the next attempt.
+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", SyncStatusRunning).Error; err != nil {
+ t.Fatalf("mark running: %v", err)
+ }
+ attempts, failed, err = dao.HandleTransientFailure(ctx, "task-1", "", "timeout", "transient", 4)
+ if err != nil {
+ t.Fatalf("HandleTransientFailure: %v", err)
+ }
+ if attempts != int64(i) {
+ t.Fatalf("attempts = %d, want %d", attempts, i)
+ }
+ if i < 4 && failed {
+ t.Fatalf("failed = true on attempt %d, want false", i)
+ }
+ }
+ if !failed {
+ t.Fatalf("failed = false, want true after 4 transient attempts")
+ }
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != SyncStatusFail {
+ t.Fatalf("status = %s, want fail", task.Status)
+ }
+ if task.ErrorCount != 7 {
+ t.Fatalf("error_count = %d, want 7 (total across classes)", task.ErrorCount)
+ }
+ if task.RetryCount != 4 {
+ t.Fatalf("retry_count = %d, want 4", task.RetryCount)
+ }
+}
diff --git a/internal/entity/connector.go b/internal/entity/connector.go
index c3b0b8c3f4..c4be15f605 100644
--- a/internal/entity/connector.go
+++ b/internal/entity/connector.go
@@ -105,21 +105,29 @@ func (Connector2Kb) TableName() string {
// SyncLogs sync logs model
type SyncLogs struct {
- ID string `gorm:"column:id;primaryKey;size:32" json:"id"`
- ConnectorID string `gorm:"column:connector_id;size:32;index" json:"connector_id"`
- TaskType string `gorm:"column:task_type;size:32;not null;default:sync;index" json:"task_type"`
- Status string `gorm:"column:status;size:128;not null;index" json:"status"`
- FromBeginning *string `gorm:"column:from_beginning;size:1" json:"from_beginning,omitempty"`
- NewDocsIndexed int64 `gorm:"column:new_docs_indexed;default:0" json:"new_docs_indexed"`
- TotalDocsIndexed int64 `gorm:"column:total_docs_indexed;default:0" json:"total_docs_indexed"`
- DocsRemovedFromIndex int64 `gorm:"column:docs_removed_from_index;default:0" json:"docs_removed_from_index"`
- ErrorMsg string `gorm:"column:error_msg;type:longtext;not null" json:"error_msg"`
- ErrorCount int64 `gorm:"column:error_count;default:0" json:"error_count"`
- FullExceptionTrace *string `gorm:"column:full_exception_trace;type:longtext" json:"full_exception_trace,omitempty"`
- TimeStarted *time.Time `gorm:"column:time_started;index" json:"time_started,omitempty"`
- PollRangeStart *FlexibleTime `gorm:"column:poll_range_start;type:varchar(255);index" json:"poll_range_start,omitempty"`
- PollRangeEnd *FlexibleTime `gorm:"column:poll_range_end;type:varchar(255);index" json:"poll_range_end,omitempty"`
- KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`
+ ID string `gorm:"column:id;primaryKey;size:32" json:"id"`
+ ConnectorID string `gorm:"column:connector_id;size:32;index" json:"connector_id"`
+ TaskType string `gorm:"column:task_type;size:32;not null;default:sync;index" json:"task_type"`
+ Status string `gorm:"column:status;size:128;not null;index" json:"status"`
+ FromBeginning *string `gorm:"column:from_beginning;size:1" json:"from_beginning,omitempty"`
+ NewDocsIndexed int64 `gorm:"column:new_docs_indexed;default:0" json:"new_docs_indexed"`
+ TotalDocsIndexed int64 `gorm:"column:total_docs_indexed;default:0" json:"total_docs_indexed"`
+ DocsRemovedFromIndex int64 `gorm:"column:docs_removed_from_index;default:0" json:"docs_removed_from_index"`
+ ErrorMsg string `gorm:"column:error_msg;type:longtext;not null" json:"error_msg"`
+ ErrorCount int64 `gorm:"column:error_count;default:0" json:"error_count"`
+ // ErrorClass identifies the retry class of the most recent failure; it is
+ // empty before the task's first failure.
+ ErrorClass string `gorm:"column:error_class;size:32;default:''" json:"error_class"`
+ // RetryCount is the number of consecutive failures for ErrorClass. It
+ // resets to 1 whenever the failure class changes so one class's retries
+ // never consume another class's budget. ErrorCount keeps the total
+ // failure count for diagnostics.
+ RetryCount int64 `gorm:"column:retry_count;default:0" json:"retry_count"`
+ FullExceptionTrace *string `gorm:"column:full_exception_trace;type:longtext" json:"full_exception_trace,omitempty"`
+ TimeStarted *time.Time `gorm:"column:time_started;index" json:"time_started,omitempty"`
+ PollRangeStart *FlexibleTime `gorm:"column:poll_range_start;type:varchar(255);index" json:"poll_range_start,omitempty"`
+ PollRangeEnd *FlexibleTime `gorm:"column:poll_range_end;type:varchar(255);index" json:"poll_range_end,omitempty"`
+ KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`
BaseModel
}
diff --git a/internal/service/connector_syncer_test.go b/internal/service/connector_syncer_test.go
index b5c8fabf7c..d1540c5e3e 100644
--- a/internal/service/connector_syncer_test.go
+++ b/internal/service/connector_syncer_test.go
@@ -139,7 +139,7 @@ func TestResumeFailedSyncSchedulesOriginalTaskFromCheckpoint(t *testing.T) {
TaskType: dao.TaskTypeSync,
Status: string(entity.TaskStatusFail),
FromBeginning: stringPtr("1"),
- ErrorMsg: "sync task failed after 3 transient retries: unexpected EOF",
+ ErrorMsg: "sync task failed after 2 retries, last error: unexpected EOF",
ErrorCount: 3,
}).Error; err != nil {
t.Fatalf("insert sync log: %v", err)
diff --git a/internal/syncer/README.md b/internal/syncer/README.md
index d1b8c9a325..31c233d84e 100644
--- a/internal/syncer/README.md
+++ b/internal/syncer/README.md
@@ -213,7 +213,7 @@ Supplementary details (beyond the diagram):
- `OpenSync`'s `WindowEnd` is fixed when the task starts; a re-run reuses the same window from the checkpoint state.
- Batch checkpoints are only saved after the batch job returns successfully; a failed batch is reprocessed on the next run.
-- Per-document failures first go through document-level retries with `ItemRetryCount = 3`; task-level transient errors then go through task rescheduling with `maxTransientTaskRetries = 3`.
+- Per-document failures first go through document-level retries with `ItemRetryCount = 3`; task-level execution errors are then retried — whitelisted transient errors 3 times, other errors 2 times.
- A failed claim `Ack`s the current message and re-publishes the wake-up after 3 seconds if the task is still scheduled.
- Connector/KB lock contention calls `RescheduleClaimed` and re-publishes the wake-up after 3 seconds.
- `Ack` only confirms the current NATS message has been handled; it does not mean the task succeeded.
@@ -476,10 +476,10 @@ Requirements when implementing checkpoints:
- If only `SourceID` is saved, the next run must be able to re-enumerate and skip data before that ID.
- Do not commit checkpoints inside the connector; checkpoints are only saved by the runner after a batch succeeds.
-Transient-error retries work on two layers:
+Retries work on two layers:
- Document level: `SyncRunner.processDocumentWithRetry` retries per-document failures matching `service.IsRetryable(err)` with exponential backoff.
-- Task level: `TaskWorker` retries tasks for transient sync errors such as timeout, 429, 5xx, and connection reset, up to `maxTransientTaskRetries` times.
+- Task level: `TaskWorker` retries every task execution error (user or system cancellation excluded). Whitelisted transient errors (timeout, 429, 5xx, connection reset, ...) are retried 3 times; all other errors are retried 2 times. When the retry budget is exhausted, the task is marked FAIL and its error message states how many retries were made and the last error.
> [!WARNING]
> Checkpoints are only saved by the runner after a batch job succeeds. Connectors must not persist "how far sync has progressed" themselves in `NextBatch` or `Fetch`; otherwise failed re-runs diverge from the runner's commit point.
@@ -492,7 +492,7 @@ Task-execution validation:
- `TaskCoordinator.Execute` calls `connector.Validate(ctx)` at the start of every task.
- This validates the saved configuration.
-- Failure leads to task failure or a transient-error retry.
+- Failure leads to a task-level retry (transient errors 3 times, others 2 times), then task failure.
Test-connection validation:
diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.go
index 77dfb3db42..b1d364f4ca 100644
--- a/internal/syncer/connector/rest_api.go
+++ b/internal/syncer/connector/rest_api.go
@@ -717,8 +717,8 @@ func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {
case status >= 400 && status < 500 && status != http.StatusTooManyRequests:
return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API request failed with non-retriable client error status %d", status)}
case status >= 500:
- // The "http <status>" wording lets the syncer's task-level retry
- // classifier (isTransientSyncError) recognize exhausted server errors.
+ // Keep the "http <status>" wording so the failure surfaces clearly in
+ // the sync task error message.
return nil, fmt.Errorf("REST API request failed with http %d", status)
}
diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go
index 9d2a7d29bc..78d64cff62 100644
--- a/internal/syncer/syncer_test.go
+++ b/internal/syncer/syncer_test.go
@@ -1165,7 +1165,7 @@ func TestSyncRunnerResultWaitHonorsCancel(t *testing.T) {
func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {
db := setupSyncerDB(t)
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
- _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(2), "retry_count": int64(2), "error_class": "non_transient"}).Error
taskDAO := dao.NewSyncTaskDAO(db)
taskService := service.NewSyncTaskService(taskDAO)
connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}
@@ -1180,6 +1180,12 @@ func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {
if task.Status != dao.SyncStatusFail {
t.Fatalf("status = %s, want fail", task.Status)
}
+ if task.ErrorCount != 3 {
+ t.Fatalf("error_count = %d, want 3", task.ErrorCount)
+ }
+ if !strings.Contains(task.ErrorMsg, "failed after 2 retries") || !strings.Contains(task.ErrorMsg, "boom") {
+ t.Fatalf("error_msg = %q", task.ErrorMsg)
+ }
if task.PollRangeEnd != nil {
t.Fatalf("poll_range_end advanced on failure")
}
@@ -1211,11 +1217,40 @@ func TestTransientFailureReschedulesTask(t *testing.T) {
}
}
-// TestTransientFailureFailsAfterThreeRetries verifies retryable errors become terminal after the retry budget.
+// TestNonTransientFailureReschedulesTask verifies every task execution error
+// (not only "transient" ones) keeps the same task scheduled for retry.
+func TestNonTransientFailureReschedulesTask(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}
+ sink := &fakeSink{errBySourceID: map[string]error{"bad": errors.New("boom")}}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinator(taskDAO, taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusSchedule {
+ t.Fatalf("status = %s, want schedule", task.Status)
+ }
+ if task.ErrorCount != 1 {
+ t.Fatalf("error_count = %d, want 1", task.ErrorCount)
+ }
+ if !strings.Contains(task.ErrorMsg, "boom") {
+ t.Fatalf("error_msg = %q, want boom", task.ErrorMsg)
+ }
+}
+
+// TestTransientFailureFailsAfterThreeRetries verifies whitelisted (transient)
+// errors become terminal after their retry budget: 3 retries plus the final attempt.
func TestTransientFailureFailsAfterThreeRetries(t *testing.T) {
db := setupSyncerDB(t)
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
- if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(2)}).Error; err != nil {
+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(3), "retry_count": int64(3), "error_class": "transient"}).Error; err != nil {
t.Fatalf("mark running: %v", err)
}
taskDAO := dao.NewSyncTaskDAO(db)
@@ -1231,14 +1266,52 @@ func TestTransientFailureFailsAfterThreeRetries(t *testing.T) {
if task.Status != dao.SyncStatusFail {
t.Fatalf("status = %s, want fail", task.Status)
}
- if task.ErrorCount != 3 {
- t.Fatalf("error_count = %d, want 3", task.ErrorCount)
+ if task.ErrorCount != 4 {
+ t.Fatalf("error_count = %d, want 4", task.ErrorCount)
}
- if !strings.Contains(task.ErrorMsg, "failed after 3 transient retries") || !strings.Contains(task.ErrorMsg, "unexpected EOF") {
+ if !strings.Contains(task.ErrorMsg, "failed after 3 retries") || !strings.Contains(task.ErrorMsg, "unexpected EOF") {
t.Fatalf("error_msg = %q", task.ErrorMsg)
}
}
+// TestErrorClassSwitchResetsRetryBudget verifies a task that switches failure
+// classes starts a fresh retry budget: failures accumulated under one class
+// must not consume the retries of a different class.
+func TestErrorClassSwitchResetsRetryBudget(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ // The task already exhausted the non-transient budget (3 failures), then
+ // hits a transient error with its own larger budget.
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{
+ "status": dao.SyncStatusRunning,
+ "error_count": int64(3),
+ "retry_count": int64(3),
+ "error_class": "non_transient",
+ }).Error
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ connector := &connectormock.Connector{SyncErrAt: 1} // unexpected EOF (transient)
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinator(taskDAO, taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusSchedule {
+ t.Fatalf("status = %s, want schedule (fresh transient budget)", task.Status)
+ }
+ if task.ErrorCount != 4 {
+ t.Fatalf("error_count = %d, want 4 (total across classes)", task.ErrorCount)
+ }
+ if task.RetryCount != 1 {
+ t.Fatalf("retry_count = %d, want 1 (reset on class change)", task.RetryCount)
+ }
+ if task.ErrorClass != "transient" {
+ t.Fatalf("error_class = %q, want transient", task.ErrorClass)
+ }
+}
+
// TestTransientFetchFailureSavesCompletedBatchCheckpoint verifies source failures do not discard completed batch progress.
func TestTransientFetchFailureSavesCompletedBatchCheckpoint(t *testing.T) {
db := setupSyncerDB(t)
diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.go
index 6bd2f522eb..f2cb6667db 100644
--- a/internal/syncer/task_worker.go
+++ b/internal/syncer/task_worker.go
@@ -19,7 +19,6 @@ package syncer
import (
"context"
"errors"
- "fmt"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/service"
@@ -151,29 +150,19 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
ackEnvelope(envelope)
return
}
- if isTransientSyncError(err) {
- attempts, failed, transientErr := w.taskDAO.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(err), maxTransientTaskRetries)
- if transientErr != nil {
- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {
- common.Warn("syncer task reschedule failed after transient failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
- }
- nackEnvelope(envelope)
- return
- }
- logTransientSyncRetry(taskContext, attempts, failed, err)
- if !failed {
- w.scheduleRetry(ctx, taskContext.Task.ID, transientRetryDelay(attempts))
- }
- ackEnvelope(envelope)
- return
- }
- if failErr := w.taskDAO.FailTask(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(fmt.Errorf("sync task failed: %w", err)), 1); failErr != nil {
+ maxRetries := maxTaskRetries(err)
+ attempts, failed, transientErr := w.taskDAO.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(err), taskErrorClass(err), maxRetries)
+ if transientErr != nil {
if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {
- common.Warn("syncer task reschedule failed after terminal failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
+ common.Warn("syncer task reschedule failed after failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))
}
nackEnvelope(envelope)
return
}
+ logSyncRetry(taskContext, attempts, failed, maxRetries, err)
+ if !failed {
+ w.scheduleRetry(ctx, taskContext.Task.ID, transientRetryDelay(attempts))
+ }
ackEnvelope(envelope)
return
}
@@ -235,7 +224,8 @@ func syncTaskErrorMessage(err error) string {
return err.Error()
}
-// transientRetryDelay return retry delay
+// transientRetryDelay returns the exponential backoff before the next task
+// retry: 10s, 20s, 40s, ... capped at 320s.
func transientRetryDelay(attempts int64) time.Duration {
if attempts < 1 {
attempts = 1
@@ -245,13 +235,13 @@ func transientRetryDelay(attempts int64) time.Duration {
if shift > 5 {
shift = 5
}
- return time.Duration(1<<shift) * 30 * time.Second
+ return time.Duration(1<<shift) * 10 * time.Second
}
-func logTransientSyncRetry(taskContext dao.SyncTaskContext, attempts int64, failed bool, err error) {
- message := "sync task transient retry scheduled"
+func logSyncRetry(taskContext dao.SyncTaskContext, attempts int64, failed bool, maxRetries int64, err error) {
+ message := "sync task retry scheduled"
if failed {
- message = "sync task failed after transient retries"
+ message = "sync task failed after retries"
}
common.Warn(
message,
@@ -260,7 +250,7 @@ func logTransientSyncRetry(taskContext dao.SyncTaskContext, attempts int64, fail
zap.String("kb_id", taskContext.Knowledgebase.ID),
zap.String("source", taskContext.Connector.Source),
zap.Int64("attempts", attempts),
- zap.Int64("max_retries", maxTransientTaskRetries),
+ zap.Int64("max_retries", maxRetries),
zap.Error(err),
)
}
diff --git a/internal/syncer/transient_error.go b/internal/syncer/transient_error.go
index 3c6047e071..16a2ac0119 100644
--- a/internal/syncer/transient_error.go
+++ b/internal/syncer/transient_error.go
@@ -24,10 +24,40 @@ import (
"strings"
)
-// maxTransientTaskRetries max retry time when network error occur
-const maxTransientTaskRetries int64 = 3
+// maxTransientTaskRetries is the total attempt budget for whitelisted
+// (transient) task errors: 3 retries plus the final attempt.
+const maxTransientTaskRetries int64 = 4
-// isTransientSyncError reports whether a task-level sync error is worth retrying.
+// maxNonTransientTaskRetries is the total attempt budget for non-whitelisted
+// task errors: 2 retries plus the final attempt.
+const maxNonTransientTaskRetries int64 = 3
+
+// Error class identifiers persisted in sync_logs.error_class. Each class
+// carries its own retry budget so one class's failures never consume another.
+const (
+ taskErrorClassTransient = "transient"
+ taskErrorClassNonTransient = "non_transient"
+)
+
+// taskErrorClass returns the persisted retry-class identifier for a task error.
+func taskErrorClass(err error) string {
+ if isTransientSyncError(err) {
+ return taskErrorClassTransient
+ }
+ return taskErrorClassNonTransient
+}
+
+// maxTaskRetries returns the total attempt budget for a task error:
+// whitelisted (transient) errors get more retries than other errors.
+func maxTaskRetries(err error) int64 {
+ if isTransientSyncError(err) {
+ return maxTransientTaskRetries
+ }
+ return maxNonTransientTaskRetries
+}
+
+// isTransientSyncError reports whether a task-level sync error is worth
+// extra retries.
func isTransientSyncError(err error) bool {
if err == nil {
return false