RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): retry sync task errors with per-class budgets (#19790)
mkaaada97f4c882b03
diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.goindex ae0c7f30e7..8d791a2e7c 100644--- a/internal/dao/sync_task.go+++ b/internal/dao/sync_task.go@@ -294,8 +294,12 @@ func (d *SyncTaskDAO) FailTask(ctx context.Context, taskID, connectorID, message})}-// HandleTransientFailure retries a running task until maxRetries is reached.-func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connectorID, message string, maxRetries int64) (int64, bool, error) {+// HandleTransientFailure retries a running task until maxRetries for its error+// class is reached. errorClass identifies the current failure class; the+// class-scoped retry counter resets whenever the class changes, so failures of+// one class never consume another class's retry budget. ErrorCount is kept as+// the total failure count for diagnostics.+func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connectorID, message, errorClass string, maxRetries int64) (int64, bool, error) {var attempts int64var failed boolerr := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {@@ -311,7 +315,11 @@ func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connecreturn err}- attempts = task.ErrorCount + 1+ attempts = task.RetryCount + 1+ if task.ErrorClass != errorClass {+ // A different failure class starts a fresh retry budget.+ attempts = 1+ }status := SyncStatusScheduleconnectorStatus := SyncStatusScheduleerrorMsg := message@@ -319,7 +327,7 @@ func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connecfailed = truestatus = SyncStatusFailconnectorStatus = SyncStatusFail- errorMsg = fmt.Sprintf("sync task failed after %d transient retries: %s", maxRetries, message)+ errorMsg = fmt.Sprintf("sync task failed after %d retries, last error: %s", attempts-1, message)}if err := tx.Model(&entity.SyncLogs{}).@@ -327,7 +335,9 @@ func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connecUpdates(map[string]any{"status": status,"error_msg": errorMsg,- "error_count": attempts,+ "error_count": task.ErrorCount + 1,+ "retry_count": attempts,+ "error_class": errorClass,}).Error; err != nil {return err}diff --git a/internal/dao/sync_task_test.go b/internal/dao/sync_task_test.gonew file mode 100644index 0000000000..5d5b607f39--- /dev/null+++ b/internal/dao/sync_task_test.go@@ -0,0 +1,141 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "fmt"+ "strings"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/entity"+)++// setupSyncTaskTestDB initializes an in-memory SQLite database for SyncTask DAO tests.+func setupSyncTaskTestDB(t *testing.T) *gorm.DB {+ t.Helper()++ dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.NewReplacer("/", "_", " ", "_").Replace(t.Name()))+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{+ TranslateError: true,+ })+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("failed to get sql DB: %v", err)+ }+ sqlDB.SetMaxOpenConns(1)++ if err := db.AutoMigrate(&entity.SyncLogs{}); err != nil {+ t.Fatalf("failed to migrate SyncLogs: %v", err)+ }+ return db+}++func insertRunningSyncTask(t *testing.T, db *gorm.DB, taskID string, errorCount, retryCount int64, errorClass string) {+ t.Helper()+ if err := db.Create(&entity.SyncLogs{+ ID: taskID,+ ConnectorID: "conn-1",+ KbID: "kb-1",+ TaskType: TaskTypeSync,+ Status: SyncStatusRunning,+ ErrorCount: errorCount,+ RetryCount: retryCount,+ ErrorClass: errorClass,+ }).Error; err != nil {+ t.Fatalf("insert task: %v", err)+ }+}++// TestHandleTransientFailureUsesClassScopedRetryBudget verifies the retry+// comparison uses a counter scoped to the current error class: failures under+// one class must not consume another class's budget, and ErrorCount keeps the+// total failure count for diagnostics.+func TestHandleTransientFailureUsesClassScopedRetryBudget(t *testing.T) {+ db := setupSyncTaskTestDB(t)+ dao := NewSyncTaskDAO(db)+ ctx := context.Background()++ // The task already exhausted the non-transient budget (3 failures), then+ // hits a transient error whose budget (4) must start fresh.+ insertRunningSyncTask(t, db, "task-1", 3, 3, "non_transient")+ attempts, failed, err := dao.HandleTransientFailure(ctx, "task-1", "", "boom", "transient", 4)+ if err != nil {+ t.Fatalf("HandleTransientFailure: %v", err)+ }+ if failed {+ t.Fatalf("failed = true, want false (transient budget must reset)")+ }+ if attempts != 1 {+ t.Fatalf("attempts = %d, want 1", attempts)+ }+ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != SyncStatusSchedule {+ t.Fatalf("status = %s, want schedule", task.Status)+ }+ if task.ErrorCount != 4 {+ t.Fatalf("error_count = %d, want 4 (diagnostics total)", task.ErrorCount)+ }+ if task.RetryCount != 1 {+ t.Fatalf("retry_count = %d, want 1", task.RetryCount)+ }+ if task.ErrorClass != "transient" {+ t.Fatalf("error_class = %q, want transient", task.ErrorClass)+ }++ // Same class keeps accumulating and fails once the class budget is spent.+ for i := 2; i <= 4; i++ {+ // The worker re-claims the scheduled task before the next attempt.+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", SyncStatusRunning).Error; err != nil {+ t.Fatalf("mark running: %v", err)+ }+ attempts, failed, err = dao.HandleTransientFailure(ctx, "task-1", "", "timeout", "transient", 4)+ if err != nil {+ t.Fatalf("HandleTransientFailure: %v", err)+ }+ if attempts != int64(i) {+ t.Fatalf("attempts = %d, want %d", attempts, i)+ }+ if i < 4 && failed {+ t.Fatalf("failed = true on attempt %d, want false", i)+ }+ }+ if !failed {+ t.Fatalf("failed = false, want true after 4 transient attempts")+ }+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != SyncStatusFail {+ t.Fatalf("status = %s, want fail", task.Status)+ }+ if task.ErrorCount != 7 {+ t.Fatalf("error_count = %d, want 7 (total across classes)", task.ErrorCount)+ }+ if task.RetryCount != 4 {+ t.Fatalf("retry_count = %d, want 4", task.RetryCount)+ }+}diff --git a/internal/entity/connector.go b/internal/entity/connector.goindex c3b0b8c3f4..c4be15f605 100644--- a/internal/entity/connector.go+++ b/internal/entity/connector.go@@ -105,21 +105,29 @@ func (Connector2Kb) TableName() string {// SyncLogs sync logs modeltype SyncLogs struct {- ID string `gorm:"column:id;primaryKey;size:32" json:"id"`- ConnectorID string `gorm:"column:connector_id;size:32;index" json:"connector_id"`- TaskType string `gorm:"column:task_type;size:32;not null;default:sync;index" json:"task_type"`- Status string `gorm:"column:status;size:128;not null;index" json:"status"`- FromBeginning *string `gorm:"column:from_beginning;size:1" json:"from_beginning,omitempty"`- NewDocsIndexed int64 `gorm:"column:new_docs_indexed;default:0" json:"new_docs_indexed"`- TotalDocsIndexed int64 `gorm:"column:total_docs_indexed;default:0" json:"total_docs_indexed"`- DocsRemovedFromIndex int64 `gorm:"column:docs_removed_from_index;default:0" json:"docs_removed_from_index"`- ErrorMsg string `gorm:"column:error_msg;type:longtext;not null" json:"error_msg"`- ErrorCount int64 `gorm:"column:error_count;default:0" json:"error_count"`- FullExceptionTrace *string `gorm:"column:full_exception_trace;type:longtext" json:"full_exception_trace,omitempty"`- TimeStarted *time.Time `gorm:"column:time_started;index" json:"time_started,omitempty"`- PollRangeStart *FlexibleTime `gorm:"column:poll_range_start;type:varchar(255);index" json:"poll_range_start,omitempty"`- PollRangeEnd *FlexibleTime `gorm:"column:poll_range_end;type:varchar(255);index" json:"poll_range_end,omitempty"`- KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`+ ID string `gorm:"column:id;primaryKey;size:32" json:"id"`+ ConnectorID string `gorm:"column:connector_id;size:32;index" json:"connector_id"`+ TaskType string `gorm:"column:task_type;size:32;not null;default:sync;index" json:"task_type"`+ Status string `gorm:"column:status;size:128;not null;index" json:"status"`+ FromBeginning *string `gorm:"column:from_beginning;size:1" json:"from_beginning,omitempty"`+ NewDocsIndexed int64 `gorm:"column:new_docs_indexed;default:0" json:"new_docs_indexed"`+ TotalDocsIndexed int64 `gorm:"column:total_docs_indexed;default:0" json:"total_docs_indexed"`+ DocsRemovedFromIndex int64 `gorm:"column:docs_removed_from_index;default:0" json:"docs_removed_from_index"`+ ErrorMsg string `gorm:"column:error_msg;type:longtext;not null" json:"error_msg"`+ ErrorCount int64 `gorm:"column:error_count;default:0" json:"error_count"`+ // ErrorClass identifies the retry class of the most recent failure; it is+ // empty before the task's first failure.+ ErrorClass string `gorm:"column:error_class;size:32;default:''" json:"error_class"`+ // RetryCount is the number of consecutive failures for ErrorClass. It+ // resets to 1 whenever the failure class changes so one class's retries+ // never consume another class's budget. ErrorCount keeps the total+ // failure count for diagnostics.+ RetryCount int64 `gorm:"column:retry_count;default:0" json:"retry_count"`+ FullExceptionTrace *string `gorm:"column:full_exception_trace;type:longtext" json:"full_exception_trace,omitempty"`+ TimeStarted *time.Time `gorm:"column:time_started;index" json:"time_started,omitempty"`+ PollRangeStart *FlexibleTime `gorm:"column:poll_range_start;type:varchar(255);index" json:"poll_range_start,omitempty"`+ PollRangeEnd *FlexibleTime `gorm:"column:poll_range_end;type:varchar(255);index" json:"poll_range_end,omitempty"`+ KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`BaseModel}diff --git a/internal/service/connector_syncer_test.go b/internal/service/connector_syncer_test.goindex b5c8fabf7c..d1540c5e3e 100644--- a/internal/service/connector_syncer_test.go+++ b/internal/service/connector_syncer_test.go@@ -139,7 +139,7 @@ func TestResumeFailedSyncSchedulesOriginalTaskFromCheckpoint(t *testing.T) {TaskType: dao.TaskTypeSync,Status: string(entity.TaskStatusFail),FromBeginning: stringPtr("1"),- ErrorMsg: "sync task failed after 3 transient retries: unexpected EOF",+ ErrorMsg: "sync task failed after 2 retries, last error: unexpected EOF",ErrorCount: 3,}).Error; err != nil {t.Fatalf("insert sync log: %v", err)diff --git a/internal/syncer/README.md b/internal/syncer/README.mdindex d1b8c9a325..31c233d84e 100644--- a/internal/syncer/README.md+++ b/internal/syncer/README.md@@ -213,7 +213,7 @@ Supplementary details (beyond the diagram):- `OpenSync`'s `WindowEnd` is fixed when the task starts; a re-run reuses the same window from the checkpoint state.- Batch checkpoints are only saved after the batch job returns successfully; a failed batch is reprocessed on the next run.-- Per-document failures first go through document-level retries with `ItemRetryCount = 3`; task-level transient errors then go through task rescheduling with `maxTransientTaskRetries = 3`.+- Per-document failures first go through document-level retries with `ItemRetryCount = 3`; task-level execution errors are then retried — whitelisted transient errors 3 times, other errors 2 times.- A failed claim `Ack`s the current message and re-publishes the wake-up after 3 seconds if the task is still scheduled.- Connector/KB lock contention calls `RescheduleClaimed` and re-publishes the wake-up after 3 seconds.- `Ack` only confirms the current NATS message has been handled; it does not mean the task succeeded.@@ -476,10 +476,10 @@ Requirements when implementing checkpoints:- If only `SourceID` is saved, the next run must be able to re-enumerate and skip data before that ID.- Do not commit checkpoints inside the connector; checkpoints are only saved by the runner after a batch succeeds.-Transient-error retries work on two layers:+Retries work on two layers:- Document level: `SyncRunner.processDocumentWithRetry` retries per-document failures matching `service.IsRetryable(err)` with exponential backoff.-- Task level: `TaskWorker` retries tasks for transient sync errors such as timeout, 429, 5xx, and connection reset, up to `maxTransientTaskRetries` times.+- Task level: `TaskWorker` retries every task execution error (user or system cancellation excluded). Whitelisted transient errors (timeout, 429, 5xx, connection reset, ...) are retried 3 times; all other errors are retried 2 times. When the retry budget is exhausted, the task is marked FAIL and its error message states how many retries were made and the last error.> [!WARNING]> Checkpoints are only saved by the runner after a batch job succeeds. Connectors must not persist "how far sync has progressed" themselves in `NextBatch` or `Fetch`; otherwise failed re-runs diverge from the runner's commit point.@@ -492,7 +492,7 @@ Task-execution validation:- `TaskCoordinator.Execute` calls `connector.Validate(ctx)` at the start of every task.- This validates the saved configuration.-- Failure leads to task failure or a transient-error retry.+- Failure leads to a task-level retry (transient errors 3 times, others 2 times), then task failure.Test-connection validation:diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.goindex 77dfb3db42..b1d364f4ca 100644--- a/internal/syncer/connector/rest_api.go+++ b/internal/syncer/connector/rest_api.go@@ -717,8 +717,8 @@ func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {case status >= 400 && status < 500 && status != http.StatusTooManyRequests:return nil, &ConnectorValidationError{Message: fmt.Sprintf("REST API request failed with non-retriable client error status %d", status)}case status >= 500:- // The "http <status>" wording lets the syncer's task-level retry- // classifier (isTransientSyncError) recognize exhausted server errors.+ // Keep the "http <status>" wording so the failure surfaces clearly in+ // the sync task error message.return nil, fmt.Errorf("REST API request failed with http %d", status)}diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.goindex 9d2a7d29bc..78d64cff62 100644--- a/internal/syncer/syncer_test.go+++ b/internal/syncer/syncer_test.go@@ -1165,7 +1165,7 @@ func TestSyncRunnerResultWaitHonorsCancel(t *testing.T) {func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {db := setupSyncerDB(t)insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)- _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(2), "retry_count": int64(2), "error_class": "non_transient"}).ErrortaskDAO := dao.NewSyncTaskDAO(db)taskService := service.NewSyncTaskService(taskDAO)connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}@@ -1180,6 +1180,12 @@ func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {if task.Status != dao.SyncStatusFail {t.Fatalf("status = %s, want fail", task.Status)}+ if task.ErrorCount != 3 {+ t.Fatalf("error_count = %d, want 3", task.ErrorCount)+ }+ if !strings.Contains(task.ErrorMsg, "failed after 2 retries") || !strings.Contains(task.ErrorMsg, "boom") {+ t.Fatalf("error_msg = %q", task.ErrorMsg)+ }if task.PollRangeEnd != nil {t.Fatalf("poll_range_end advanced on failure")}@@ -1211,11 +1217,40 @@ func TestTransientFailureReschedulesTask(t *testing.T) {}}-// TestTransientFailureFailsAfterThreeRetries verifies retryable errors become terminal after the retry budget.+// TestNonTransientFailureReschedulesTask verifies every task execution error+// (not only "transient" ones) keeps the same task scheduled for retry.+func TestNonTransientFailureReschedulesTask(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}+ sink := &fakeSink{errBySourceID: map[string]error{"bad": errors.New("boom")}}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinator(taskDAO, taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusSchedule {+ t.Fatalf("status = %s, want schedule", task.Status)+ }+ if task.ErrorCount != 1 {+ t.Fatalf("error_count = %d, want 1", task.ErrorCount)+ }+ if !strings.Contains(task.ErrorMsg, "boom") {+ t.Fatalf("error_msg = %q, want boom", task.ErrorMsg)+ }+}++// TestTransientFailureFailsAfterThreeRetries verifies whitelisted (transient)+// errors become terminal after their retry budget: 3 retries plus the final attempt.func TestTransientFailureFailsAfterThreeRetries(t *testing.T) {db := setupSyncerDB(t)insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)- if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(2)}).Error; err != nil {+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(3), "retry_count": int64(3), "error_class": "transient"}).Error; err != nil {t.Fatalf("mark running: %v", err)}taskDAO := dao.NewSyncTaskDAO(db)@@ -1231,14 +1266,52 @@ func TestTransientFailureFailsAfterThreeRetries(t *testing.T) {if task.Status != dao.SyncStatusFail {t.Fatalf("status = %s, want fail", task.Status)}- if task.ErrorCount != 3 {- t.Fatalf("error_count = %d, want 3", task.ErrorCount)+ if task.ErrorCount != 4 {+ t.Fatalf("error_count = %d, want 4", task.ErrorCount)}- if !strings.Contains(task.ErrorMsg, "failed after 3 transient retries") || !strings.Contains(task.ErrorMsg, "unexpected EOF") {+ if !strings.Contains(task.ErrorMsg, "failed after 3 retries") || !strings.Contains(task.ErrorMsg, "unexpected EOF") {t.Fatalf("error_msg = %q", task.ErrorMsg)}}+// TestErrorClassSwitchResetsRetryBudget verifies a task that switches failure+// classes starts a fresh retry budget: failures accumulated under one class+// must not consume the retries of a different class.+func TestErrorClassSwitchResetsRetryBudget(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ // The task already exhausted the non-transient budget (3 failures), then+ // hits a transient error with its own larger budget.+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{+ "status": dao.SyncStatusRunning,+ "error_count": int64(3),+ "retry_count": int64(3),+ "error_class": "non_transient",+ }).Error+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ connector := &connectormock.Connector{SyncErrAt: 1} // unexpected EOF (transient)+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinator(taskDAO, taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusSchedule {+ t.Fatalf("status = %s, want schedule (fresh transient budget)", task.Status)+ }+ if task.ErrorCount != 4 {+ t.Fatalf("error_count = %d, want 4 (total across classes)", task.ErrorCount)+ }+ if task.RetryCount != 1 {+ t.Fatalf("retry_count = %d, want 1 (reset on class change)", task.RetryCount)+ }+ if task.ErrorClass != "transient" {+ t.Fatalf("error_class = %q, want transient", task.ErrorClass)+ }+}+// TestTransientFetchFailureSavesCompletedBatchCheckpoint verifies source failures do not discard completed batch progress.func TestTransientFetchFailureSavesCompletedBatchCheckpoint(t *testing.T) {db := setupSyncerDB(t)diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.goindex 6bd2f522eb..f2cb6667db 100644--- a/internal/syncer/task_worker.go+++ b/internal/syncer/task_worker.go@@ -19,7 +19,6 @@ package syncerimport ("context""errors"- "fmt""ragflow/internal/common""ragflow/internal/dao""ragflow/internal/service"@@ -151,29 +150,19 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {ackEnvelope(envelope)return}- if isTransientSyncError(err) {- attempts, failed, transientErr := w.taskDAO.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(err), maxTransientTaskRetries)- if transientErr != nil {- if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {- common.Warn("syncer task reschedule failed after transient failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))- }- nackEnvelope(envelope)- return- }- logTransientSyncRetry(taskContext, attempts, failed, err)- if !failed {- w.scheduleRetry(ctx, taskContext.Task.ID, transientRetryDelay(attempts))- }- ackEnvelope(envelope)- return- }- if failErr := w.taskDAO.FailTask(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(fmt.Errorf("sync task failed: %w", err)), 1); failErr != nil {+ maxRetries := maxTaskRetries(err)+ attempts, failed, transientErr := w.taskDAO.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, syncTaskErrorMessage(err), taskErrorClass(err), maxRetries)+ if transientErr != nil {if err = w.rescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {- common.Warn("syncer task reschedule failed after terminal failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))+ common.Warn("syncer task reschedule failed after failure handling error", zap.String("task_id", taskContext.Task.ID), zap.Error(err))}nackEnvelope(envelope)return}+ logSyncRetry(taskContext, attempts, failed, maxRetries, err)+ if !failed {+ w.scheduleRetry(ctx, taskContext.Task.ID, transientRetryDelay(attempts))+ }ackEnvelope(envelope)return}@@ -235,7 +224,8 @@ func syncTaskErrorMessage(err error) string {return err.Error()}-// transientRetryDelay return retry delay+// transientRetryDelay returns the exponential backoff before the next task+// retry: 10s, 20s, 40s, ... capped at 320s.func transientRetryDelay(attempts int64) time.Duration {if attempts < 1 {attempts = 1@@ -245,13 +235,13 @@ func transientRetryDelay(attempts int64) time.Duration {if shift > 5 {shift = 5}- return time.Duration(1<<shift) * 30 * time.Second+ return time.Duration(1<<shift) * 10 * time.Second}-func logTransientSyncRetry(taskContext dao.SyncTaskContext, attempts int64, failed bool, err error) {- message := "sync task transient retry scheduled"+func logSyncRetry(taskContext dao.SyncTaskContext, attempts int64, failed bool, maxRetries int64, err error) {+ message := "sync task retry scheduled"if failed {- message = "sync task failed after transient retries"+ message = "sync task failed after retries"}common.Warn(message,@@ -260,7 +250,7 @@ func logTransientSyncRetry(taskContext dao.SyncTaskContext, attempts int64, failzap.String("kb_id", taskContext.Knowledgebase.ID),zap.String("source", taskContext.Connector.Source),zap.Int64("attempts", attempts),- zap.Int64("max_retries", maxTransientTaskRetries),+ zap.Int64("max_retries", maxRetries),zap.Error(err),)}diff --git a/internal/syncer/transient_error.go b/internal/syncer/transient_error.goindex 3c6047e071..16a2ac0119 100644--- a/internal/syncer/transient_error.go+++ b/internal/syncer/transient_error.go@@ -24,10 +24,40 @@ import ("strings")-// maxTransientTaskRetries max retry time when network error occur-const maxTransientTaskRetries int64 = 3+// maxTransientTaskRetries is the total attempt budget for whitelisted+// (transient) task errors: 3 retries plus the final attempt.+const maxTransientTaskRetries int64 = 4-// isTransientSyncError reports whether a task-level sync error is worth retrying.+// maxNonTransientTaskRetries is the total attempt budget for non-whitelisted+// task errors: 2 retries plus the final attempt.+const maxNonTransientTaskRetries int64 = 3++// Error class identifiers persisted in sync_logs.error_class. Each class+// carries its own retry budget so one class's failures never consume another.+const (+ taskErrorClassTransient = "transient"+ taskErrorClassNonTransient = "non_transient"+)++// taskErrorClass returns the persisted retry-class identifier for a task error.+func taskErrorClass(err error) string {+ if isTransientSyncError(err) {+ return taskErrorClassTransient+ }+ return taskErrorClassNonTransient+}++// maxTaskRetries returns the total attempt budget for a task error:+// whitelisted (transient) errors get more retries than other errors.+func maxTaskRetries(err error) int64 {+ if isTransientSyncError(err) {+ return maxTransientTaskRetries+ }+ return maxNonTransientTaskRetries+}++// isTransientSyncError reports whether a task-level sync error is worth+// extra retries.func isTransientSyncError(err error) bool {if err == nil {return false