Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat[Go]: resuming transmission from the point of interruption during data source synchronisation (#18176)

Haruko38696f7c39753a9
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index e7c6fab1db..900913e5e3 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -633,15 +633,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
func runSyncer(ctx context.Context, cancel context.CancelFunc, args *serverArgs) error {
globalConfig := server.GetConfig()
syncerConfig := globalConfig.GetSyncerConfig()
- fileSyncer := syncer.NewSyncer(syncerConfig.MaxConcurrentSyncs, time.Duration(syncerConfig.SyncInterval)*time.Second)
+ fileSyncer := syncer.NewSyncer(syncerConfig.MaxConcurrentSyncs)
- go func() {
- err := fileSyncer.Start()
- if err != nil {
- common.Error("Failed to initialize file syncer", err)
- return
- }
- }()
+ if err := fileSyncer.StartContext(ctx); err != nil {
+ common.Error("Failed to initialize file syncer", err)
+ return err
+ }
common.Info("\n _______ __ _____\n" +
" / ____(_) /__ / ___/__ ______ ________ _____\n" +
diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml
index db12d36bcb..fa181efb82 100644
--- a/conf/service_conf.yaml
+++ b/conf/service_conf.yaml
@@ -72,7 +72,6 @@ ingestor:
compiler_pool_size: 0
file_syncer:
max_concurrent_syncs: 5
- sync_interval: 3
user_default_llm:
default_models:
embedding_model:
diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.template
index 745b0def57..b60e1317c4 100644
--- a/docker/service_conf.yaml.template
+++ b/docker/service_conf.yaml.template
@@ -90,7 +90,6 @@ ingestor:
compiler_pool_size: 0
file_syncer:
max_concurrent_syncs: 5
- sync_interval: 3
user_default_llm:
default_models:
embedding_model:
diff --git a/internal/dao/connector.go b/internal/dao/connector.go
index e7dfd5b5fe..fff493bf7e 100644
--- a/internal/dao/connector.go
+++ b/internal/dao/connector.go
@@ -25,6 +25,7 @@ import (
"time"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
// ConnectorDAO connector data access object
@@ -105,21 +106,26 @@ type DatasetConnectorLink struct {
}
// LinkDatasetConnectors syncs connector2kb rows for a dataset.
-func (dao *ConnectorDAO) LinkDatasetConnectors(ctx context.Context, db *gorm.DB, kbID string, connectors []DatasetConnectorLink) error {
- return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+func (dao *ConnectorDAO) LinkDatasetConnectors(ctx context.Context, db *gorm.DB, kbID string, connectors []DatasetConnectorLink) ([]string, error) {
+ var taskIDs []string
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var kb entity.Knowledgebase
- if err := db.WithContext(ctx).Select("tenant_id").Where("id = ? AND status = ?", kbID, string(entity.StatusValid)).First(&kb).Error; err != nil {
+ if err := tx.WithContext(ctx).Select("tenant_id").Where("id = ? AND status = ?", kbID, string(entity.StatusValid)).First(&kb).Error; err != nil {
return err
}
- return dao.LinkDatasetConnectorsTx(ctx, tx, kbID, kb.TenantID, connectors)
+ var err error
+ taskIDs, err = dao.LinkDatasetConnectorsTx(ctx, tx, kbID, kb.TenantID, connectors)
+ return err
})
+ return taskIDs, err
}
// LinkDatasetConnectorsTx syncs connector2kb rows using the caller's transaction.
-func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.DB, kbID, tenantID string, connectors []DatasetConnectorLink) error {
+func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.DB, kbID, tenantID string, connectors []DatasetConnectorLink) ([]string, error) {
+ var taskIDs []string
var existing []entity.Connector2Kb
if err := tx.WithContext(ctx).Where("kb_id = ?", kbID).Find(&existing).Error; err != nil {
- return err
+ return nil, err
}
oldConnectorIDs := make(map[string]entity.Connector2Kb, len(existing))
@@ -138,16 +144,16 @@ func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.D
var fullConnector entity.Connector
if err := tx.WithContext(ctx).Where("id = ? AND tenant_id = ?", connector.ID, tenantID).First(&fullConnector).Error; err != nil {
if IsNotFoundErr(err) {
- return fmt.Errorf("%w: %s", errConnectorNotAccessible, connector.ID)
+ return nil, fmt.Errorf("%w: %s", errConnectorNotAccessible, connector.ID)
}
- return err
+ return nil, err
}
if _, ok := oldConnectorIDs[connector.ID]; ok {
if err := tx.WithContext(ctx).Model(&entity.Connector2Kb{}).
Where("connector_id = ? AND kb_id = ?", connector.ID, kbID).
Update("auto_parse", autoParse).Error; err != nil {
- return err
+ return nil, err
}
continue
}
@@ -158,17 +164,21 @@ func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.D
KbID: kbID,
AutoParse: autoParse,
}).Error; err != nil {
- return err
+ return nil, err
}
- if _, err := scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypeSync, true); err != nil {
- return err
+ taskID, err := scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypeSync, true)
+ if err != nil {
+ return nil, err
}
+ taskIDs = append(taskIDs, taskID)
if connectorConfigBool(fullConnector.Config, "sync_deleted_files") {
- if _, err := scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypePrune, false); err != nil {
- return err
+ taskID, err = scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypePrune, false)
+ if err != nil {
+ return nil, err
}
+ taskIDs = append(taskIDs, taskID)
}
}
@@ -178,16 +188,16 @@ func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.D
}
if err := tx.WithContext(ctx).Where("kb_id = ? AND connector_id = ?", kbID, connectorID).
Delete(&entity.Connector2Kb{}).Error; err != nil {
- return err
+ return nil, err
}
if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).
Where("connector_id = ? AND kb_id = ? AND status IN ?", connectorID, kbID, []string{string(entity.TaskStatusSchedule), string(entity.TaskStatusRunning)}).
Update("status", string(entity.TaskStatusCancel)).Error; err != nil {
- return err
+ return nil, err
}
}
- return nil
+ return taskIDs, nil
}
// GetByID get connector by ID
@@ -270,9 +280,15 @@ func (dao *ConnectorDAO) ListDocumentsByKBAndSourceType(ctx context.Context, db
}
// RebuildConnector replaces old connector documents with scheduled sync tasks.
-func (dao *ConnectorDAO) RebuildConnector(ctx context.Context, db *gorm.DB, connector *entity.Connector, kbID string, documents []*entity.Document) ([]string, error) {
- taskIDs := []string{}
+func (dao *ConnectorDAO) RebuildConnector(ctx context.Context, db *gorm.DB, connector *entity.Connector, kbID string, documents []*entity.Document) ([]string, []string, error) {
+ var taskIDs []string
+ var oldSyncTaskIDs []string
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).
+ Where("connector_id = ? AND kb_id = ? AND task_type = ?", connector.ID, kbID, connectorTaskTypeSync).
+ Pluck("id", &oldSyncTaskIDs).Error; err != nil {
+ return err
+ }
if err := tx.WithContext(ctx).Where("connector_id = ? AND kb_id = ?", connector.ID, kbID).Delete(&entity.SyncLogs{}).Error; err != nil {
return err
}
@@ -351,7 +367,38 @@ func (dao *ConnectorDAO) RebuildConnector(ctx context.Context, db *gorm.DB, conn
}
return nil
})
- return taskIDs, err
+ return taskIDs, oldSyncTaskIDs, err
+}
+
+// ResumeFailedSyncTask schedules one failed sync task for checkpoint resume.
+func (dao *ConnectorDAO) ResumeFailedSyncTask(ctx context.Context, db *gorm.DB, connectorID, kbID, taskID string) error {
+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var task entity.SyncLogs
+ query := tx.WithContext(ctx)
+ if tx.Dialector.Name() != "sqlite" {
+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})
+ }
+ if err := query.
+ Where("id = ? AND connector_id = ? AND kb_id = ? AND task_type = ? AND status = ?", taskID, connectorID, kbID, connectorTaskTypeSync, string(entity.TaskStatusFail)).
+ First(&task).Error; err != nil {
+ return err
+ }
+
+ now := time.Now().Local()
+ if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).
+ Where("id = ? AND status = ?", taskID, string(entity.TaskStatusFail)).
+ Updates(map[string]any{
+ "status": string(entity.TaskStatusSchedule),
+ "error_msg": "",
+ "error_count": int64(0),
+ "time_started": &now,
+ }).Error; err != nil {
+ return err
+ }
+ return tx.WithContext(ctx).Model(&entity.Connector{}).
+ Where("id = ?", connectorID).
+ Update("status", string(entity.TaskStatusSchedule)).Error
+ })
}
const (
diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.go
index 00825859b1..2d2bc953bc 100644
--- a/internal/dao/sync_task.go
+++ b/internal/dao/sync_task.go
@@ -19,6 +19,7 @@ package dao
import (
"context"
"errors"
+ "fmt"
"ragflow/internal/entity"
"ragflow/internal/utility"
"time"
@@ -92,8 +93,16 @@ type dueSyncTaskRow struct {
ConnectorConfig entity.JSONMap `gorm:"column:connector_config"`
}
-// ListDueTasks returns due schedule tasks across SYNC and PRUNE.
-func (d *SyncTaskDAO) ListDueTasks(ctx context.Context, now time.Time, limit int) ([]entity.SyncLogs, error) {
+// ScheduledSyncTask contains one scheduled task and its connector scheduling settings.
+type ScheduledSyncTask struct {
+ entity.SyncLogs
+ ConnectorRefreshFreq int64
+ ConnectorPruneFreq int64
+ ConnectorConfig entity.JSONMap
+}
+
+// ListScheduledTasks returns scheduled tasks with connector scheduling settings.
+func (d *SyncTaskDAO) ListScheduledTasks(ctx context.Context, limit int) ([]ScheduledSyncTask, error) {
var rows []dueSyncTaskRow
if err := d.db.WithContext(ctx).
Model(&entity.SyncLogs{}).
@@ -103,38 +112,41 @@ func (d *SyncTaskDAO) ListDueTasks(ctx context.Context, now time.Time, limit int
Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").
Where("sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).
Order("sync_logs.update_time DESC").
- Limit(limit * 4).
+ Limit(limit).
Scan(&rows).Error; err != nil {
return nil, err
}
-
- tasks := make([]entity.SyncLogs, 0, limit)
+ tasks := make([]ScheduledSyncTask, 0, len(rows))
for _, row := range rows {
- if !isDue(row.SyncLogs, row.ConnectorRefreshFreq, row.ConnectorPruneFreq, row.ConnectorConfig, now) {
- continue
- }
- tasks = append(tasks, row.SyncLogs)
- if len(tasks) >= limit {
- break
- }
+ tasks = append(tasks, ScheduledSyncTask{
+ SyncLogs: row.SyncLogs,
+ ConnectorRefreshFreq: row.ConnectorRefreshFreq,
+ ConnectorPruneFreq: row.ConnectorPruneFreq,
+ ConnectorConfig: row.ConnectorConfig,
+ })
}
return tasks, nil
}
-// ListStartupTasks returns scheduled tasks that should be published during NATS startup reconciliation.
-func (d *SyncTaskDAO) ListStartupTasks(ctx context.Context, limit int) ([]entity.SyncLogs, error) {
- var tasks []entity.SyncLogs
- err := d.db.WithContext(ctx).
+// GetScheduledTask returns one scheduled task with connector scheduling settings.
+func (d *SyncTaskDAO) GetScheduledTask(ctx context.Context, taskID string) (ScheduledSyncTask, error) {
+ var row dueSyncTaskRow
+ if err := d.db.WithContext(ctx).
Model(&entity.SyncLogs{}).
- Select("sync_logs.*").
+ Select("sync_logs.*, connector.refresh_freq AS connector_refresh_freq, connector.prune_freq AS connector_prune_freq, connector.config AS connector_config").
Joins("JOIN connector ON sync_logs.connector_id = connector.id").
Joins("JOIN connector2kb ON sync_logs.connector_id = connector2kb.connector_id AND sync_logs.kb_id = connector2kb.kb_id").
Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").
- Where("sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).
- Order("sync_logs.update_time DESC").
- Limit(limit).
- Find(&tasks).Error
- return tasks, err
+ Where("sync_logs.id = ? AND sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", taskID, SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).
+ First(&row).Error; err != nil {
+ return ScheduledSyncTask{}, err
+ }
+ return ScheduledSyncTask{
+ SyncLogs: row.SyncLogs,
+ ConnectorRefreshFreq: row.ConnectorRefreshFreq,
+ ConnectorPruneFreq: row.ConnectorPruneFreq,
+ ConnectorConfig: row.ConnectorConfig,
+ }, nil
}
// ClaimTask conditionally marks a scheduled task as running.
@@ -268,9 +280,55 @@ func (d *SyncTaskDAO) FailTask(ctx context.Context, taskID, connectorID, message
})
}
+// HandleTransientFailure retries a running task until maxRetries is reached.
+func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connectorID, message string, maxRetries int64) (int64, bool, error) {
+ var attempts int64
+ var failed bool
+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var task entity.SyncLogs
+ query := tx.WithContext(ctx)
+ if tx.Dialector.Name() != "sqlite" {
+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})
+ }
+ if err := query.Where("id = ? AND status = ?", taskID, SyncStatusRunning).First(&task).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil
+ }
+ return err
+ }
+
+ attempts = task.ErrorCount + 1
+ status := SyncStatusSchedule
+ connectorStatus := SyncStatusSchedule
+ errorMsg := message
+ if attempts >= maxRetries {
+ failed = true
+ status = SyncStatusFail
+ connectorStatus = SyncStatusFail
+ errorMsg = fmt.Sprintf("sync task failed after %d transient retries: %s", maxRetries, message)
+ }
+
+ if err := tx.Model(&entity.SyncLogs{}).
+ Where("id = ? AND status = ?", taskID, SyncStatusRunning).
+ Updates(map[string]any{
+ "status": status,
+ "error_msg": errorMsg,
+ "error_count": attempts,
+ }).Error; err != nil {
+ return err
+ }
+ if connectorID == "" {
+ return nil
+ }
+ return tx.Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", connectorStatus).Error
+ })
+ return attempts, failed, err
+}
+
// CompleteSyncTask marks SYNC done and creates the next schedule task.
-func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, newDocs, totalDocs, errorCount int64, errorMsg string) error {
- return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, newDocs, totalDocs, errorCount int64, errorMsg string) (string, error) {
+ var nextTaskID string
+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
result := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", taskContext.Task.ID, SyncStatusRunning).Updates(map[string]any{
"status": SyncStatusDone,
"poll_range_end": pollRangeEnd,
@@ -289,13 +347,17 @@ func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTask
return err
}
- return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypeSync, false, &pollRangeEnd, taskContext.Task.TotalDocsIndexed+totalDocs)
+ var err error
+ nextTaskID, err = createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypeSync, false, &pollRangeEnd, taskContext.Task.TotalDocsIndexed+totalDocs)
+ return err
})
+ return nextTaskID, err
}
// CompletePruneTask marks PRUNE done and creates the next schedule task.
-func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTaskContext, removed int64) error {
- return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTaskContext, removed int64) (string, error) {
+ var nextTaskID string
+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
result := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", taskContext.Task.ID, SyncStatusRunning).Updates(map[string]any{
"status": SyncStatusDone,
"docs_removed_from_index": gorm.Expr("docs_removed_from_index + ?", removed),
@@ -312,8 +374,11 @@ func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTas
if !syncConnectorConfigBool(taskContext.Connector.Config, "sync_deleted_files") {
return nil
}
- return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypePrune, false, nil, taskContext.Task.TotalDocsIndexed)
+ var err error
+ nextTaskID, err = createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypePrune, false, nil, taskContext.Task.TotalDocsIndexed)
+ return err
})
+ return nextTaskID, err
}
// RecoverStaleRunning restores timed-out running tasks to schedule.
@@ -387,10 +452,15 @@ func (d *SyncTaskDAO) RecoverRunning(ctx context.Context) (int64, error) {
connectorIDs := map[string]struct{}{}
return int64(len(rows)), d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ dueAt := time.Unix(0, 0).Local()
for _, row := range rows {
if err := tx.Model(&entity.SyncLogs{}).
Where("id = ? AND status = ?", row.ID, SyncStatusRunning).
- Update("status", SyncStatusSchedule).Error; err != nil {
+ Updates(map[string]any{
+ "status": SyncStatusSchedule,
+ "update_time": dueAt.UnixMilli(),
+ "update_date": dueAt,
+ }).Error; err != nil {
return err
}
connectorIDs[row.ConnectorID] = struct{}{}
@@ -405,7 +475,7 @@ func (d *SyncTaskDAO) RecoverRunning(ctx context.Context) (int64, error) {
}
// createScheduledTask creates the next Python-compatible scheduled task.
-func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taskType string, fromBeginning bool, pollRangeStart *time.Time, totalDocsIndexed int64) error {
+func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taskType string, fromBeginning bool, pollRangeStart *time.Time, totalDocsIndexed int64) (string, error) {
var lockRow entity.Connector2Kb
query := tx.WithContext(ctx)
if tx.Dialector.Name() != "sqlite" {
@@ -414,17 +484,19 @@ func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, ta
if err := query.
Where("connector_id = ? AND kb_id = ?", connectorID, kbID).
First(&lockRow).Error; err != nil {
- return err
+ return "", err
}
- var existing int64
- if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).
+ var existing entity.SyncLogs
+ err := tx.WithContext(ctx).
Where("connector_id = ? AND kb_id = ? AND task_type = ? AND status = ?", connectorID, kbID, taskType, SyncStatusSchedule).
- Count(&existing).Error; err != nil {
- return err
+ Order("update_time DESC").
+ First(&existing).Error
+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ return "", err
}
- if existing > 0 {
- return nil
+ if err == nil {
+ return existing.ID, nil
}
reindex := "0"
@@ -436,10 +508,11 @@ func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, ta
if err := tx.WithContext(ctx).Model(&entity.Connector{}).
Where("id = ?", connectorID).
Update("status", SyncStatusSchedule).Error; err != nil {
- return err
+ return "", err
}
- return tx.WithContext(ctx).Create(&entity.SyncLogs{
- ID: utility.GenerateToken(),
+ taskID := utility.GenerateToken()
+ return taskID, tx.WithContext(ctx).Create(&entity.SyncLogs{
+ ID: taskID,
ConnectorID: connectorID,
KbID: kbID,
TaskType: taskType,
@@ -452,29 +525,6 @@ func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, ta
}).Error
}
-// isDue applies refresh_freq and prune_freq scheduling semantics.
-func isDue(task entity.SyncLogs, refreshFreq, pruneFreq int64, config map[string]any, now time.Time) bool {
- if task.UpdateDate == nil {
- return true
- }
- var freqMinutes int64
- switch task.TaskType {
- case TaskTypeSync:
- freqMinutes = refreshFreq
- case TaskTypePrune:
- if !syncConnectorConfigBool(config, "sync_deleted_files") {
- return false
- }
- freqMinutes = pruneFreq
- default:
- return false
- }
- if freqMinutes <= 0 {
- return true
- }
- return task.UpdateDate.Before(now.Add(-time.Duration(freqMinutes) * time.Minute))
-}
-
// syncConnectorConfigBool reads a Python JSON bool/string flag.
func syncConnectorConfigBool(config map[string]any, key string) bool {
value, ok := config[key]
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index d809699898..6bab124d74 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -44,9 +44,10 @@ type NatsEngine struct {
knowledgeCompileConsumer jetstream.Consumer
kv jetstream.KeyValue
- syncerStream jetstream.Stream
- syncerConsumer jetstream.Consumer
- syncerMu sync.Mutex
+ syncerStream jetstream.Stream
+ syncerConsumer jetstream.PushConsumer
+ syncCheckpointKV jetstream.KeyValue
+ syncerMu sync.Mutex
}
func NewNatsEngine(host string, port int) *NatsEngine {
diff --git a/internal/engine/nats/syncer.go b/internal/engine/nats/syncer.go
index 5b109d5171..83cf0179df 100644
--- a/internal/engine/nats/syncer.go
+++ b/internal/engine/nats/syncer.go
@@ -25,9 +25,10 @@ import (
"time"
"ragflow/internal/common"
+ syncerconnector "ragflow/internal/syncer/connector"
- "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
+ "go.uber.org/zap"
)
const (
@@ -36,7 +37,11 @@ const (
syncerStreamName = "RAGFLOW_SYNC_TASKS"
syncerConsumerName = "RAGFLOW_SYNCER_CONSUMER"
+ syncCheckpointBucket = "RAGFLOW_SYNC_CHECKPOINTS"
+ syncerDeliverSubject = "deliver.syncer.RAGFLOW"
+ syncerDeliverGroup = "RAGFLOW_SYNCER_WORKERS"
syncerSubjectPattern = "sync.tasks.>"
+ syncCheckpointTTL = 7 * 24 * time.Hour
)
// InitSyncerStream creates the datasource syncer task stream.
@@ -79,7 +84,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {
return nil
}
-// InitSyncerConsumer creates the durable pull consumer for syncer tasks.
+// InitSyncerConsumer creates the durable push consumer for syncer tasks.
func (n *NatsEngine) InitSyncerConsumer() error {
n.syncerMu.Lock()
defer n.syncerMu.Unlock()
@@ -89,28 +94,109 @@ func (n *NatsEngine) InitSyncerConsumer() error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
- consumer, err := n.syncerStream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
- Name: syncerConsumerName,
- Durable: syncerConsumerName,
- AckPolicy: jetstream.AckExplicitPolicy,
- MaxDeliver: 16,
- MaxAckPending: 1024 * 128,
- FilterSubject: syncerSubjectPattern,
- })
- if err != nil {
- if strings.Contains(err.Error(), "max waiting can not be updated") {
- consumer, err = n.syncerStream.Consumer(ctx, syncerConsumerName)
- if err != nil {
- return fmt.Errorf("syncer: get existing consumer: %w", err)
+ config := syncerPushConsumerConfig()
+ consumer, err := n.syncerStream.CreateOrUpdatePushConsumer(ctx, config)
+ if err != nil && shouldRecreateSyncerConsumer(ctx, n.syncerStream, config, err) {
+ common.Warn("syncer replacing incompatible NATS consumer", zap.String("consumer", syncerConsumerName), zap.Error(err), zap.Bool("delete", true))
+ if deleteErr := n.syncerStream.DeleteConsumer(ctx, syncerConsumerName); deleteErr != nil {
+ if !strings.Contains(strings.ToLower(deleteErr.Error()), "not found") {
+ return fmt.Errorf("syncer: replace existing consumer: %w", deleteErr)
}
- } else {
- return fmt.Errorf("syncer: create consumer: %w", err)
}
+ consumer, err = n.syncerStream.CreateOrUpdatePushConsumer(ctx, config)
+ }
+ if err != nil {
+ return fmt.Errorf("syncer: create push consumer: %w", err)
}
n.syncerConsumer = consumer
return nil
}
+// InitSyncCheckpoints creates the KV bucket backing running sync task checkpoints.
+func (n *NatsEngine) InitSyncCheckpoints() error {
+ n.syncerMu.Lock()
+ defer n.syncerMu.Unlock()
+ return n.initSyncCheckpointsLocked()
+}
+
+func (n *NatsEngine) initSyncCheckpointsLocked() error {
+ if n.jetStream == nil {
+ return fmt.Errorf("syncer checkpoint: jetStream not initialized")
+ }
+ if n.syncCheckpointKV != nil {
+ return nil
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ kv, err := n.jetStream.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+ Bucket: syncCheckpointBucket,
+ Description: "running datasource sync checkpoints",
+ History: 1,
+ TTL: syncCheckpointTTL,
+ MaxValueSize: 1024 * 1024,
+ Storage: jetstream.FileStorage,
+ })
+ if err != nil {
+ return fmt.Errorf("syncer checkpoint: create kv: %w", err)
+ }
+ n.syncCheckpointKV = kv
+ return nil
+}
+
+func syncerPushConsumerConfig() jetstream.ConsumerConfig {
+ return jetstream.ConsumerConfig{
+ Name: syncerConsumerName,
+ Durable: syncerConsumerName,
+ AckPolicy: jetstream.AckExplicitPolicy,
+ MaxDeliver: 16,
+ MaxAckPending: 1024,
+ FilterSubject: syncerSubjectPattern,
+ DeliverSubject: syncerDeliverSubject,
+ DeliverGroup: syncerDeliverGroup,
+ }
+}
+
+func shouldRecreateSyncerConsumer(ctx context.Context, stream jetstream.Stream, desired jetstream.ConsumerConfig, err error) bool {
+ var jsErr jetstream.JetStreamError
+ if !errors.As(err, &jsErr) || jsErr.APIError() == nil || jsErr.APIError().ErrorCode != jetstream.JSErrCodeConsumerCreate {
+ return false
+ }
+
+ existing, inspectErr := syncerExistingConsumerConfig(ctx, stream)
+ if inspectErr != nil {
+ common.Warn("syncer consumer replacement skipped", zap.String("consumer", syncerConsumerName), zap.Error(err), zap.NamedError("inspect_error", inspectErr), zap.Bool("delete", false))
+ return false
+ }
+ if existing == nil {
+ common.Warn("syncer consumer replacement skipped", zap.String("consumer", syncerConsumerName), zap.Error(err), zap.Bool("delete", false))
+ return false
+ }
+ return syncerConsumerConfigMismatch(*existing, desired)
+}
+
+func syncerExistingConsumerConfig(ctx context.Context, stream jetstream.Stream) (*jetstream.ConsumerConfig, error) {
+ lister := stream.ListConsumers(ctx)
+ for info := range lister.Info() {
+ if info != nil && info.Name == syncerConsumerName {
+ config := info.Config
+ return &config, nil
+ }
+ }
+ return nil, lister.Err()
+}
+
+func syncerConsumerConfigMismatch(existing, desired jetstream.ConsumerConfig) bool {
+ return existing.Durable != desired.Durable ||
+ existing.Name != desired.Name ||
+ existing.AckPolicy != desired.AckPolicy ||
+ existing.MaxDeliver != desired.MaxDeliver ||
+ existing.MaxAckPending != desired.MaxAckPending ||
+ existing.FilterSubject != desired.FilterSubject ||
+ existing.DeliverSubject != desired.DeliverSubject ||
+ existing.DeliverGroup != desired.DeliverGroup
+}
+
// PublishSyncerTask publishes one sync_logs task wake-up.
func (n *NatsEngine) PublishSyncerTask(taskID string) error {
if err := n.InitSyncerStream(); err != nil {
@@ -129,29 +215,116 @@ func (n *NatsEngine) PublishSyncerTask(taskID string) error {
return err
}
-// FetchSyncerTasks pulls syncer task messages from JetStream.
-func (n *NatsEngine) FetchSyncerTasks(batchSize int) ([]common.TaskHandle, error) {
+// PublishSyncerTaskWakeup publishes a non-deduplicated wake-up for an existing sync_logs task.
+func (n *NatsEngine) PublishSyncerTaskWakeup(taskID string) error {
+ if err := n.InitSyncerStream(); err != nil {
+ return err
+ }
+
+ payload, err := json.Marshal(common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeSyncer})
+ if err != nil {
+ return err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ msgID := fmt.Sprintf("%s.manual-wakeup.%d", taskID, time.Now().UnixNano())
+ _, err = n.jetStream.Publish(ctx, SyncerTaskSubject, payload, jetstream.WithMsgID(msgID), jetstream.WithExpectStream(syncerStreamName))
+ return err
+}
+
+// SubscribeSyncerTasks starts push delivery for syncer task messages.
+func (n *NatsEngine) SubscribeSyncerTasks(ctx context.Context, handler func(common.TaskHandle)) error {
n.syncerMu.Lock()
consumer := n.syncerConsumer
n.syncerMu.Unlock()
+
if consumer == nil {
- return nil, fmt.Errorf("syncer: consumer not initialized")
+ return fmt.Errorf("syncer: consumer not initialized")
+ }
+
+ consumeCtx, err := consumer.Consume(func(msg jetstream.Msg) {
+ handler(NewNatsMessageHandle(msg))
+ })
+ if err != nil {
+ return err
+ }
+
+ go func() {
+ <-ctx.Done()
+ consumeCtx.Stop()
+ <-consumeCtx.Closed()
+ }()
+ return nil
+}
+
+// LoadSyncCheckpoint reads the latest running checkpoint for one sync task.
+func (n *NatsEngine) LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error) {
+ kv, err := n.syncCheckpointStore()
+ if err != nil {
+ return nil, err
}
- // fetch task from nats(jetStream)
- messages, err := consumer.Fetch(batchSize, jetstream.FetchMaxWait(1*time.Second))
+
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ entry, err := kv.Get(ctx, syncCheckpointKey(taskID))
if err != nil {
- if errors.Is(err, nats.ErrTimeout) {
+ if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
return nil, err
}
- handles := make([]common.TaskHandle, 0, batchSize)
- for msg := range messages.Messages() {
- handles = append(handles, NewNatsMessageHandle(msg))
+ var state syncerconnector.SyncCheckpointState
+ if err = json.Unmarshal(entry.Value(), &state); err != nil {
+ return nil, err
}
- if err = messages.Error(); err != nil {
- return handles, err
+ return &state, nil
+}
+
+// SaveSyncCheckpoint writes the latest running checkpoint for one sync task.
+func (n *NatsEngine) SaveSyncCheckpoint(ctx context.Context, taskID string, state syncerconnector.SyncCheckpointState) error {
+ kv, err := n.syncCheckpointStore()
+ if err != nil {
+ return err
}
- return handles, nil
+
+ // write to json
+ data, err := json.Marshal(state)
+ if err != nil {
+ return err
+ }
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ _, err = kv.Put(ctx, syncCheckpointKey(taskID), data)
+ return err
+}
+
+// DeleteSyncCheckpoint removes the running checkpoint for a completed sync task.
+func (n *NatsEngine) DeleteSyncCheckpoint(ctx context.Context, taskID string) error {
+ kv, err := n.syncCheckpointStore()
+ if err != nil {
+ return err
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ err = kv.Delete(ctx, syncCheckpointKey(taskID))
+ if errors.Is(err, jetstream.ErrKeyNotFound) {
+ return nil
+ }
+ return err
+}
+
+func (n *NatsEngine) syncCheckpointStore() (jetstream.KeyValue, error) {
+ n.syncerMu.Lock()
+ defer n.syncerMu.Unlock()
+ if err := n.initSyncCheckpointsLocked(); err != nil {
+ return nil, err
+ }
+ return n.syncCheckpointKV, nil
+}
+
+func syncCheckpointKey(taskID string) string {
+ return "task." + taskID
}
diff --git a/internal/engine/nats/syncer_test.go b/internal/engine/nats/syncer_test.go
index 03ae16adbc..5ec96d6a10 100644
--- a/internal/engine/nats/syncer_test.go
+++ b/internal/engine/nats/syncer_test.go
@@ -1,7 +1,9 @@
package nats
import (
+ "context"
"net"
+ syncerconnector "ragflow/internal/syncer/connector"
"testing"
"time"
@@ -10,8 +12,8 @@ import (
"github.com/nats-io/nats-server/v2/server"
)
-// TestSyncerTaskStreamPublishesAndFetches verifies the dedicated syncer stream path.
-func TestSyncerTaskStreamPublishesAndFetches(t *testing.T) {
+// TestSyncerTaskStreamPublishesAndSubscribes verifies the dedicated syncer stream path.
+func TestSyncerTaskStreamPublishesAndSubscribes(t *testing.T) {
engine := setupSyncerNATSEngine(t)
if err := engine.InitSyncerStream(); err != nil {
t.Fatalf("InitSyncerStream: %v", err)
@@ -19,26 +21,73 @@ func TestSyncerTaskStreamPublishesAndFetches(t *testing.T) {
if err := engine.InitSyncerConsumer(); err != nil {
t.Fatalf("InitSyncerConsumer: %v", err)
}
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ received := make(chan common.TaskHandle, 1)
+ if err := engine.SubscribeSyncerTasks(ctx, func(handle common.TaskHandle) {
+ received <- handle
+ }); err != nil {
+ t.Fatalf("SubscribeSyncerTasks: %v", err)
+ }
if err := engine.PublishSyncerTask("task-1"); err != nil {
t.Fatalf("PublishSyncerTask: %v", err)
}
- handles, err := engine.FetchSyncerTasks(1)
- if err != nil {
- t.Fatalf("FetchSyncerTasks: %v", err)
- }
- if len(handles) != 1 {
- t.Fatalf("handles len = %d, want 1", len(handles))
+ var handle common.TaskHandle
+ select {
+ case handle = <-received:
+ case <-time.After(5 * time.Second):
+ t.Fatal("timed out waiting for syncer task")
}
- message := handles[0].GetMessage()
+ message := handle.GetMessage()
if message.TaskID != "task-1" || message.TaskType != common.TaskTypeSyncer {
t.Fatalf("message = %+v", message)
}
- if err := handles[0].Ack(); err != nil {
+ if err := handle.Ack(); err != nil {
t.Fatalf("Ack: %v", err)
}
}
+// TestSyncCheckpointKVSaveLoadDelete verifies the syncer checkpoint KV lifecycle.
+func TestSyncCheckpointKVSaveLoadDelete(t *testing.T) {
+ engine := setupSyncerNATSEngine(t)
+ now := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
+ state := syncerconnector.SyncCheckpointState{
+ Version: 1,
+ TaskID: "task-1",
+ ConnectorID: "conn-1",
+ KBID: "kb-1",
+ WindowEnd: now,
+ NextCommitSeq: 2,
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &now, SourceID: "source-1"},
+ }
+ if err := engine.SaveSyncCheckpoint(t.Context(), "task-1", state); err != nil {
+ t.Fatalf("SaveSyncCheckpoint: %v", err)
+ }
+
+ loaded, err := engine.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("LoadSyncCheckpoint: %v", err)
+ }
+ if loaded == nil || loaded.Checkpoint == nil {
+ t.Fatalf("loaded checkpoint = %+v, want value", loaded)
+ }
+ if loaded.TaskID != "task-1" || loaded.Checkpoint.Cursor != "cursor-1" || loaded.Checkpoint.SourceID != "source-1" {
+ t.Fatalf("loaded checkpoint = %+v", loaded)
+ }
+
+ if err = engine.DeleteSyncCheckpoint(t.Context(), "task-1"); err != nil {
+ t.Fatalf("DeleteSyncCheckpoint: %v", err)
+ }
+ loaded, err = engine.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("LoadSyncCheckpoint after delete: %v", err)
+ }
+ if loaded != nil {
+ t.Fatalf("loaded checkpoint after delete = %+v, want nil", loaded)
+ }
+}
+
func setupSyncerNATSEngine(t *testing.T) *NatsEngine {
t.Helper()
opts := &server.Options{
diff --git a/internal/handler/connector.go b/internal/handler/connector.go
index ce1aac2bcc..d71e22ef00 100644
--- a/internal/handler/connector.go
+++ b/internal/handler/connector.go
@@ -38,6 +38,7 @@ type connectorServiceIface interface {
ListLog(ctx context.Context, connectorID, userID string, page, pageSize int) ([]*entity.ConnectorSyncLog, int64, common.ErrorCode, error)
DeleteConnector(ctx context.Context, connectorID, userID string) (bool, common.ErrorCode, error)
RebuildConnector(ctx context.Context, connectorID, userID, kbID string) (bool, common.ErrorCode, error)
+ ResumeFailedSync(ctx context.Context, connectorID, userID string, req *service.ResumeFailedSyncRequest) (bool, common.ErrorCode, error)
TestConnector(ctx context.Context, connectorID, userID string) error
UpdateConnector(ctx context.Context, connectorID, userID string, req *service.UpdateConnectorRequest) (*entity.Connector, common.ErrorCode, error)
StartGoogleWebOAuth(ctx context.Context, userID, source string, req *service.StartGoogleWebOAuthRequest) (*service.StartGoogleWebOAuthResponse, common.ErrorCode, error)
@@ -380,6 +381,36 @@ func (h *ConnectorHandler) RebuildConnector(c *gin.Context) {
common.SuccessWithData(c, ok, "success")
}
+// ResumeFailedSync resumes a failed connector sync task from checkpoint.
+// @Summary Resume Failed Connector Sync
+// @Description Resume a failed connector sync task from its saved checkpoint
+// @Tags connector
+// @Accept json
+// @Produce json
+// @Success 200 {object} map[string]interface{}
+// @Router /connector/:connector_id/resume-failed-sync [post]
+func (h *ConnectorHandler) ResumeFailedSync(c *gin.Context) {
+ user, errorCode, errorMessage := GetUser(c)
+ if errorCode != common.CodeSuccess {
+ common.ErrorWithCode(c, errorCode, errorMessage)
+ return
+ }
+
+ var req service.ResumeFailedSyncRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ common.ResponseWithCodeData(c, common.CodeDataError, nil, "required argument is missing: kb_id or task_id")
+ return
+ }
+
+ ctx := c.Request.Context()
+ ok, code, err := h.connectorService.ResumeFailedSync(ctx, c.Param("connector_id"), user.ID, &req)
+ if err != nil {
+ common.ErrorWithCode(c, code, err.Error())
+ return
+ }
+ common.SuccessWithData(c, ok, "success")
+}
+
func (h *ConnectorHandler) StartGoogleWebOAuth(c *gin.Context) {
user, errorCode, errorMessage := GetUser(c)
if errorCode != common.CodeSuccess {
diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.go
index 676b9411c7..57080390b4 100644
--- a/internal/handler/connector_test.go
+++ b/internal/handler/connector_test.go
@@ -119,6 +119,13 @@ func (s fakeConnectorService) RebuildConnector(context.Context, string, string,
return true, common.CodeSuccess, nil
}
+func (s fakeConnectorService) ResumeFailedSync(context.Context, string, string, *service.ResumeFailedSyncRequest) (bool, common.ErrorCode, error) {
+ if s.err != nil {
+ return false, s.code, s.err
+ }
+ return true, common.CodeSuccess, nil
+}
+
func TestConnectorHandlerStartBoxWebOAuth(t *testing.T) {
gin.SetMode(gin.TestMode)
diff --git a/internal/router/router.go b/internal/router/router.go
index fd2dc699c2..0dc12816c4 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -635,6 +635,7 @@ func (r *Router) Setup(engine *gin.Engine) {
connectors.GET("/:connector_id/logs", r.connectorHandler.ListLogs)
connectors.DELETE("/:connector_id", r.connectorHandler.DeleteConnector)
connectors.POST("/:connector_id/rebuild", r.connectorHandler.RebuildConnector)
+ connectors.POST("/:connector_id/resume-failed-sync", r.connectorHandler.ResumeFailedSync)
connectors.POST("/:connector_id/test", r.connectorHandler.TestConnector)
}
diff --git a/internal/server/config/syncer_config.go b/internal/server/config/syncer_config.go
index 16f1911de4..4265242bb5 100644
--- a/internal/server/config/syncer_config.go
+++ b/internal/server/config/syncer_config.go
@@ -20,13 +20,11 @@ import "github.com/spf13/viper"
type SyncerConfig struct {
MaxConcurrentSyncs int `mapstructure:"max_concurrent_syncs"`
- SyncInterval int `mapstructure:"sync_interval"`
}
func (c *Config) ParseSyncerConfig(v *viper.Viper) error {
// Default Syncer config
c.syncer.MaxConcurrentSyncs = 5
- c.syncer.SyncInterval = 3
if !v.IsSet("file_syncer") {
return nil
@@ -40,10 +38,6 @@ func (c *Config) ParseSyncerConfig(v *viper.Viper) error {
c.syncer.MaxConcurrentSyncs = sub.GetInt("max_concurrent_syncs")
}
- if sub.IsSet("sync_interval") {
- c.syncer.SyncInterval = sub.GetInt("sync_interval")
- }
-
return nil
}
diff --git a/internal/service/connector.go b/internal/service/connector.go
index 17fabc2149..e3fd4133a1 100644
--- a/internal/service/connector.go
+++ b/internal/service/connector.go
@@ -29,10 +29,12 @@ import (
"net/http"
"net/url"
"ragflow/internal/engine/redis"
+ syncerconnector "ragflow/internal/syncer/connector"
"ragflow/internal/utility"
"strings"
"time"
+ "go.uber.org/zap"
"gorm.io/gorm"
"ragflow/internal/common"
@@ -91,11 +93,33 @@ type syncTaskPublisher interface {
PublishSyncerTask(taskID string) error
}
+type syncTaskWakeupPublisher interface {
+ PublishSyncerTaskWakeup(taskID string) error
+}
+
+type syncCheckpointLoader interface {
+ LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error)
+}
+
+type syncCheckpointDeleter interface {
+ DeleteSyncCheckpoint(ctx context.Context, taskID string) error
+}
+
var getSyncerTaskPublisher = func() (syncTaskPublisher, bool) {
publisher, ok := engine.GetMessageQueueEngine().(syncTaskPublisher)
return publisher, ok
}
+var getSyncCheckpointLoader = func() (syncCheckpointLoader, bool) {
+ loader, ok := engine.GetMessageQueueEngine().(syncCheckpointLoader)
+ return loader, ok
+}
+
+var getSyncCheckpointDeleter = func() (syncCheckpointDeleter, bool) {
+ deleter, ok := engine.GetMessageQueueEngine().(syncCheckpointDeleter)
+ return deleter, ok
+}
+
// NewConnectorService create connector service
func NewConnectorService() *ConnectorService {
return &ConnectorService{
@@ -124,6 +148,12 @@ type RebuildConnectorRequest struct {
KbID string `json:"kb_id"`
}
+// ResumeFailedSyncRequest resumes a failed connector sync task from checkpoint.
+type ResumeFailedSyncRequest struct {
+ KbID string `json:"kb_id"`
+ TaskID string `json:"task_id"`
+}
+
type StartGoogleWebOAuthRequest struct {
Credentials json.RawMessage `json:"credentials"`
RedirectURI string `json:"redirect_uri,omitempty"`
@@ -934,9 +964,7 @@ func (s *ConnectorService) UpdateConnector(ctx context.Context, connectorID, use
if err != nil {
return nil, common.CodeServerError, err
}
- if err = publishSyncerTasks(taskIDs); err != nil {
- return nil, common.CodeServerError, err
- }
+ publishSyncerTasks(taskIDs)
} else if isConnectorCancelStatus(req.Status) {
if err = s.cancelConnectorTasks(ctx, connectorID); err != nil {
return nil, common.CodeServerError, err
@@ -946,9 +974,7 @@ func (s *ConnectorService) UpdateConnector(ctx context.Context, connectorID, use
if err != nil {
return nil, common.CodeServerError, err
}
- if err = publishSyncerTasks(taskIDs); err != nil {
- return nil, common.CodeServerError, err
- }
+ publishSyncerTasks(taskIDs)
}
}
@@ -1006,29 +1032,127 @@ func (s *ConnectorService) RebuildConnector(ctx context.Context, connectorID, us
s.deleteConnectorDocumentChunks(ctx, connector.TenantID, kbID, documents)
- taskIDs, err := s.connectorDAO.RebuildConnector(ctx, dao.DB, connector, kbID, documents)
+ taskIDs, oldSyncTaskIDs, err := s.connectorDAO.RebuildConnector(ctx, dao.DB, connector, kbID, documents)
+ if err != nil {
+ return false, common.CodeServerError, err
+ }
+
+ if err = deleteSyncCheckpoints(ctx, oldSyncTaskIDs); err != nil {
+ common.Warn("delete sync checkpoints failed during rebuild",
+ zap.String("connector_id", connectorID), zap.Error(err))
+ }
+
+ publishSyncerTasks(taskIDs)
+ return true, common.CodeSuccess, nil
+}
+
+// ResumeFailedSync schedules a failed sync task to continue from its saved checkpoint.
+func (s *ConnectorService) ResumeFailedSync(ctx context.Context, connectorID, userID string, req *ResumeFailedSyncRequest) (bool, common.ErrorCode, error) {
+ if connectorID == "" {
+ return false, common.CodeDataError, fmt.Errorf("connector_id is required")
+ }
+ if req == nil {
+ return false, common.CodeArgumentError, fmt.Errorf("request is required")
+ }
+ // check KBid and TaskID
+ req.KbID = strings.TrimSpace(req.KbID)
+ req.TaskID = strings.TrimSpace(req.TaskID)
+ if req.KbID == "" {
+ return false, common.CodeArgumentError, fmt.Errorf("required argument is missing: kb_id")
+ }
+ if req.TaskID == "" {
+ return false, common.CodeArgumentError, fmt.Errorf("required argument is missing: task_id")
+ }
+ // get connector
+ connector, err := s.connectorDAO.GetByID(ctx, dao.DB, connectorID)
+ if err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return false, common.CodeDataError, fmt.Errorf("can't find this Connector")
+ }
+ return false, common.CodeServerError, err
+ }
+ // check access
+ canAccess, err := s.canAccessConnector(ctx, connector, userID)
+ if err != nil {
+ return false, common.CodeServerError, err
+ }
+ if !canAccess {
+ return false, common.CodeAuthenticationError, fmt.Errorf("no authorization")
+ }
+
+ // load the checkpoint
+ loader, ok := getSyncCheckpointLoader()
+ if !ok {
+ return false, common.CodeServerError, fmt.Errorf("sync checkpoint store is not configured")
+ }
+
+ checkpoint, err := loader.LoadSyncCheckpoint(ctx, req.TaskID)
if err != nil {
return false, common.CodeServerError, err
}
- if err = publishSyncerTasks(taskIDs); err != nil {
+ if checkpoint == nil || checkpoint.Checkpoint == nil {
+ return false, common.CodeDataError, fmt.Errorf("checkpoint not found, cannot resume failed sync task")
+ }
+ if checkpoint.TaskID != req.TaskID || checkpoint.ConnectorID != connectorID || checkpoint.KBID != req.KbID {
+ return false, common.CodeDataError, fmt.Errorf("checkpoint does not match connector, knowledge base, and task")
+ }
+ // resume the task
+ if err = s.connectorDAO.ResumeFailedSyncTask(ctx, dao.DB, connectorID, req.KbID, req.TaskID); err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return false, common.CodeDataError, fmt.Errorf("failed sync task not found")
+ }
+ return false, common.CodeServerError, err
+ }
+ if err = publishSyncerTaskWakeup(req.TaskID); err != nil {
return false, common.CodeServerError, err
}
return true, common.CodeSuccess, nil
}
-func publishSyncerTasks(taskIDs []string) error {
+func publishSyncerTasks(taskIDs []string) {
if len(taskIDs) == 0 {
- return nil
+ return
}
publisher, ok := getSyncerTaskPublisher()
if !ok {
- return fmt.Errorf("syncer task publisher is not configured")
+ common.Warn("syncer task publisher is not configured")
+ return
}
for _, taskID := range taskIDs {
if taskID == "" {
continue
}
if err := publisher.PublishSyncerTask(taskID); err != nil {
+ common.Warn("syncer task publish failed", zap.String("task_id", taskID), zap.Error(err))
+ }
+ }
+}
+
+// publishSyncerTaskWakeup publish syncer task wakeup message to nats
+func publishSyncerTaskWakeup(taskID string) error {
+ publisher, ok := getSyncerTaskPublisher()
+ if !ok {
+ return fmt.Errorf("syncer task publisher is not configured")
+ }
+ if wakeupPublisher, ok := publisher.(syncTaskWakeupPublisher); ok {
+ return wakeupPublisher.PublishSyncerTaskWakeup(taskID)
+ }
+ return publisher.PublishSyncerTask(taskID)
+}
+
+func deleteSyncCheckpoints(ctx context.Context, taskIDs []string) error {
+ if len(taskIDs) == 0 {
+ return nil
+ }
+ deleter, ok := getSyncCheckpointDeleter()
+ if !ok {
+ return fmt.Errorf("sync checkpoint store is not configured")
+ }
+ for _, taskID := range taskIDs {
+ if taskID == "" {
+ continue
+ }
+ if err := deleter.DeleteSyncCheckpoint(ctx, taskID); err != nil {
return err
}
}
diff --git a/internal/service/connector_syncer_test.go b/internal/service/connector_syncer_test.go
index a29824830f..2f70859500 100644
--- a/internal/service/connector_syncer_test.go
+++ b/internal/service/connector_syncer_test.go
@@ -7,10 +7,12 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ syncerconnector "ragflow/internal/syncer/connector"
)
type fakeSyncerTaskPublisher struct {
- taskIDs []string
+ taskIDs []string
+ wakeupTaskIDs []string
}
func (p *fakeSyncerTaskPublisher) PublishSyncerTask(taskID string) error {
@@ -18,6 +20,29 @@ func (p *fakeSyncerTaskPublisher) PublishSyncerTask(taskID string) error {
return nil
}
+func (p *fakeSyncerTaskPublisher) PublishSyncerTaskWakeup(taskID string) error {
+ p.wakeupTaskIDs = append(p.wakeupTaskIDs, taskID)
+ return nil
+}
+
+type fakeSyncCheckpointLoader struct {
+ state *syncerconnector.SyncCheckpointState
+ deletedIDs []string
+}
+
+func (l *fakeSyncCheckpointLoader) LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error) {
+ return l.state, nil
+}
+
+func (l *fakeSyncCheckpointLoader) DeleteSyncCheckpoint(ctx context.Context, taskID string) error {
+ l.deletedIDs = append(l.deletedIDs, taskID)
+ return nil
+}
+
+func stringPtr(value string) *string {
+ return &value
+}
+
func TestUpdateConnectorSchedulePublishesSyncerTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -86,6 +111,166 @@ func TestUpdateConnectorSchedulePublishesSyncerTask(t *testing.T) {
}
}
+func TestResumeFailedSyncSchedulesOriginalTaskFromCheckpoint(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.SyncLogs{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+ if err := db.Create(&entity.Connector{
+ ID: "conn-1",
+ TenantID: "user-1",
+ Name: "conn-1",
+ Source: "gmail",
+ InputType: "poll",
+ Config: entity.JSONMap{},
+ Status: string(entity.TaskStatusFail),
+ RefreshFreq: 0,
+ PruneFreq: 0,
+ TimeoutSecs: 60,
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+ if err := db.Create(&entity.SyncLogs{
+ ID: "task-1",
+ ConnectorID: "conn-1",
+ KbID: "kb-1",
+ TaskType: dao.TaskTypeSync,
+ Status: string(entity.TaskStatusFail),
+ FromBeginning: stringPtr("1"),
+ ErrorMsg: "sync task failed after 3 transient retries: unexpected EOF",
+ ErrorCount: 3,
+ }).Error; err != nil {
+ t.Fatalf("insert sync log: %v", err)
+ }
+
+ publisher := &fakeSyncerTaskPublisher{}
+ previousPublisher := getSyncerTaskPublisher
+ getSyncerTaskPublisher = func() (syncTaskPublisher, bool) {
+ return publisher, true
+ }
+ previousLoader := getSyncCheckpointLoader
+ store := &fakeSyncCheckpointLoader{state: &syncerconnector.SyncCheckpointState{
+ Version: 1,
+ TaskID: "task-1",
+ ConnectorID: "conn-1",
+ KBID: "kb-1",
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", SourceID: "source-1"},
+ }}
+ getSyncCheckpointLoader = func() (syncCheckpointLoader, bool) {
+ return store, true
+ }
+ t.Cleanup(func() {
+ getSyncerTaskPublisher = previousPublisher
+ getSyncCheckpointLoader = previousLoader
+ })
+
+ ok, code, err := NewConnectorService().ResumeFailedSync(context.Background(), "conn-1", "user-1", &ResumeFailedSyncRequest{KbID: "kb-1", TaskID: "task-1"})
+ if err != nil {
+ t.Fatalf("ResumeFailedSync error: %v", err)
+ }
+ if !ok || code != common.CodeSuccess {
+ t.Fatalf("ok/code = %v/%v, want true/success", ok, code)
+ }
+ if len(publisher.wakeupTaskIDs) != 1 || publisher.wakeupTaskIDs[0] != "task-1" {
+ t.Fatalf("wakeup task IDs = %v, want task-1", publisher.wakeupTaskIDs)
+ }
+ if len(publisher.taskIDs) != 0 {
+ t.Fatalf("regular publish task IDs = %v, want none", publisher.taskIDs)
+ }
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != string(entity.TaskStatusSchedule) {
+ t.Fatalf("task status = %s, want schedule", task.Status)
+ }
+ if task.ErrorCount != 0 || task.ErrorMsg != "" {
+ t.Fatalf("error count/msg = %d/%q, want reset", task.ErrorCount, task.ErrorMsg)
+ }
+ if task.FromBeginning == nil || *task.FromBeginning != "1" {
+ t.Fatalf("from_beginning = %v, want preserved full-sync task", task.FromBeginning)
+ }
+}
+
+func TestRebuildConnectorDeletesOldSyncCheckpointsBeforePublishing(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.Connector2Kb{}, &entity.Knowledgebase{}, &entity.SyncLogs{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+ if err := db.Create(&entity.Connector{
+ ID: "conn-1",
+ TenantID: "user-1",
+ Name: "conn-1",
+ Source: "gmail",
+ InputType: "poll",
+ Config: entity.JSONMap{},
+ Status: string(entity.TaskStatusFail),
+ RefreshFreq: 0,
+ PruneFreq: 0,
+ TimeoutSecs: 60,
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+ if err := db.Create(&entity.Knowledgebase{
+ ID: "kb-1",
+ TenantID: "user-1",
+ Name: "kb-1",
+ CreatedBy: "user-1",
+ EmbdID: "embd",
+ }).Error; err != nil {
+ t.Fatalf("insert kb: %v", err)
+ }
+ if err := db.Create(&entity.Connector2Kb{ID: "conn-1-kb-1", ConnectorID: "conn-1", KbID: "kb-1", AutoParse: "1"}).Error; err != nil {
+ t.Fatalf("insert connector mapping: %v", err)
+ }
+ if err := db.Create([]entity.SyncLogs{
+ {ID: "failed-sync", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusFail), ErrorMsg: ""},
+ {ID: "done-sync", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusDone), ErrorMsg: ""},
+ {ID: "failed-prune", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypePrune, Status: string(entity.TaskStatusFail), ErrorMsg: ""},
+ }).Error; err != nil {
+ t.Fatalf("insert sync logs: %v", err)
+ }
+
+ publisher := &fakeSyncerTaskPublisher{}
+ previousPublisher := getSyncerTaskPublisher
+ getSyncerTaskPublisher = func() (syncTaskPublisher, bool) {
+ return publisher, true
+ }
+ store := &fakeSyncCheckpointLoader{}
+ previousDeleter := getSyncCheckpointDeleter
+ getSyncCheckpointDeleter = func() (syncCheckpointDeleter, bool) {
+ return store, true
+ }
+ t.Cleanup(func() {
+ getSyncerTaskPublisher = previousPublisher
+ getSyncCheckpointDeleter = previousDeleter
+ })
+
+ ok, code, err := NewConnectorService().RebuildConnector(context.Background(), "conn-1", "user-1", "kb-1")
+ if err != nil {
+ t.Fatalf("RebuildConnector error: %v", err)
+ }
+ if !ok || code != common.CodeSuccess {
+ t.Fatalf("ok/code = %v/%v, want true/success", ok, code)
+ }
+ if len(publisher.taskIDs) != 1 {
+ t.Fatalf("published task IDs = %v, want one new rebuild task", publisher.taskIDs)
+ }
+ if len(store.deletedIDs) != 2 {
+ t.Fatalf("deleted checkpoint IDs = %v, want failed-sync and done-sync", store.deletedIDs)
+ }
+ deleted := map[string]bool{}
+ for _, taskID := range store.deletedIDs {
+ deleted[taskID] = true
+ }
+ if !deleted["failed-sync"] || !deleted["done-sync"] || deleted["failed-prune"] {
+ t.Fatalf("deleted checkpoint IDs = %v", store.deletedIDs)
+ }
+}
+
func TestUpdateConnectorScheduleDoesNotDuplicateRunningTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.go
index 2cd95310d5..0a2cdbee07 100644
--- a/internal/service/dataset/update.go
+++ b/internal/service/dataset/update.go
@@ -9,6 +9,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/engine/types"
"ragflow/internal/entity"
pipelinepkg "ragflow/internal/ingestion/pipeline"
@@ -170,6 +171,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
txCode := common.CodeSuccess
var updatedKB *entity.Knowledgebase
var linkedConnectors []*dao.ConnectorDatasetListItem
+ var scheduledTaskIDs []string
var pagerankUpdate *datasetPagerankUpdate
err = dao.DB.Transaction(func(tx *gorm.DB) error {
lockedKB, code, authErr := d.lockAccessibleDatasetForUpdate(tx, datasetID, tenantID)
@@ -320,7 +322,8 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
}
if connectorsProvided {
- if err = d.connectorDAO.LinkDatasetConnectorsTx(ctx, tx, lockedKB.ID, lockedKB.TenantID, connectorLinks); err != nil {
+ scheduledTaskIDs, err = d.connectorDAO.LinkDatasetConnectorsTx(ctx, tx, lockedKB.ID, lockedKB.TenantID, connectorLinks)
+ if err != nil {
if dao.IsConnectorNotAccessibleErr(err) {
txCode = common.CodeDataError
return err
@@ -350,6 +353,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
}
return nil, txCode, err
}
+ publishDatasetSyncerTasks(scheduledTaskIDs)
if pagerankUpdate != nil {
if err = d.updateDatasetPagerankChunks(ctx, *pagerankUpdate); err != nil {
@@ -379,6 +383,29 @@ func (d *DatasetService) updateDatasetPagerankChunks(ctx context.Context, update
return err
}
+type datasetSyncTaskPublisher interface {
+ PublishSyncerTask(taskID string) error
+}
+
+func publishDatasetSyncerTasks(taskIDs []string) {
+ if len(taskIDs) == 0 {
+ return
+ }
+ publisher, ok := engine.GetMessageQueueEngine().(datasetSyncTaskPublisher)
+ if !ok {
+ common.Warn("syncer task publisher is not configured")
+ return
+ }
+ for _, taskID := range taskIDs {
+ if taskID == "" {
+ continue
+ }
+ if err := publisher.PublishSyncerTask(taskID); err != nil {
+ common.Warn("syncer task publish failed", zap.String("task_id", taskID), zap.Error(err))
+ }
+ }
+}
+
func (d *DatasetService) lockAccessibleDatasetForUpdate(tx *gorm.DB, datasetID, userID string) (*entity.Knowledgebase, common.ErrorCode, error) {
var kb entity.Knowledgebase
err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
diff --git a/internal/service/sync_document_id_resolver.go b/internal/service/sync_document_id_resolver.go
index b9605badc7..68edbf66b4 100644
--- a/internal/service/sync_document_id_resolver.go
+++ b/internal/service/sync_document_id_resolver.go
@@ -42,6 +42,7 @@ type ResolvedDocumentID struct {
// DocumentStore reads existing connector documents.
type DocumentStore interface {
ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error)
+ ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error)
GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error)
}
@@ -55,6 +56,11 @@ func NewDocumentIDResolver(store DocumentStore) *DocumentIDResolver {
return &DocumentIDResolver{store: store}
}
+// ListFingerprintsBySourceType returns all fingerprints for one connector namespace.
+func (r *DocumentIDResolver) ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error) {
+ return r.store.ListFingerprintsBySourceType(ctx, kbID, sourceType)
+}
+
// Resolve chooses the legacy ID when already present in the KB.
func (r *DocumentIDResolver) Resolve(ctx context.Context, kbID, connectorID, sourceType, sourceID string) (ResolvedDocumentID, error) {
legacyID := Hash128(connectorID + ":" + sourceID)
@@ -100,6 +106,26 @@ func (s *GormDocumentStore) ListIDs(ctx context.Context, kbID, sourceType string
return result, nil
}
+// ListFingerprintsBySourceType returns all existing fingerprints for one connector namespace.
+func (s *GormDocumentStore) ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error) {
+ var docs []entity.Document
+ if err := dao.GetDB().WithContext(ctx).
+ Select("id", "content_hash").
+ Where("kb_id = ? AND source_type = ?", kbID, sourceType).
+ Find(&docs).Error; err != nil {
+ return nil, err
+ }
+ result := make(map[string]string, len(docs))
+ for _, doc := range docs {
+ if doc.ContentHash == nil {
+ result[doc.ID] = ""
+ continue
+ }
+ result[doc.ID] = *doc.ContentHash
+ }
+ return result, nil
+}
+
// GetFingerprintsByIDs returns fingerprints for the requested candidate IDs.
func (s *GormDocumentStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {
var docs []entity.Document
diff --git a/internal/service/sync_task_service.go b/internal/service/sync_task_service.go
index b3cf3e8064..83fcc26370 100644
--- a/internal/service/sync_task_service.go
+++ b/internal/service/sync_task_service.go
@@ -34,6 +34,9 @@ const (
// SyncTaskContext contains the database rows needed to execute a task.
type SyncTaskContext = dao.SyncTaskContext
+// ScheduledSyncTask contains one scheduled task and its connector schedule settings.
+type ScheduledSyncTask = dao.ScheduledSyncTask
+
// SyncStats accumulates task-level document results.
type SyncStats struct {
Added int64
@@ -79,30 +82,14 @@ func NewSyncTaskService(taskDAO *dao.SyncTaskDAO) *SyncTaskService {
return &SyncTaskService{taskDAO: taskDAO}
}
-// ListDueTasks returns due schedule tasks.
-func (s *SyncTaskService) ListDueTasks(ctx context.Context, now time.Time) ([]dao.SyncTask, error) {
- tasks, err := s.taskDAO.ListDueTasks(ctx, now, 128)
- if err != nil {
- return nil, err
- }
- out := make([]dao.SyncTask, 0, len(tasks))
- for _, task := range tasks {
- out = append(out, dao.SyncTask{SyncLogs: task})
- }
- return out, nil
+// ListScheduledTasks returns scheduled tasks for one-time timer reconciliation.
+func (s *SyncTaskService) ListScheduledTasks(ctx context.Context) ([]ScheduledSyncTask, error) {
+ return s.taskDAO.ListScheduledTasks(ctx, 4096)
}
-// ListStartupTasks returns scheduled tasks for NATS startup reconciliation.
-func (s *SyncTaskService) ListStartupTasks(ctx context.Context) ([]dao.SyncTask, error) {
- tasks, err := s.taskDAO.ListStartupTasks(ctx, 128)
- if err != nil {
- return nil, err
- }
- out := make([]dao.SyncTask, 0, len(tasks))
- for _, task := range tasks {
- out = append(out, dao.SyncTask{SyncLogs: task})
- }
- return out, nil
+// GetScheduledTask returns one scheduled task for timer registration.
+func (s *SyncTaskService) GetScheduledTask(ctx context.Context, taskID string) (ScheduledSyncTask, error) {
+ return s.taskDAO.GetScheduledTask(ctx, taskID)
}
// Claim marks a scheduled task running if no other scanner claimed it first.
@@ -144,14 +131,23 @@ func (s *SyncTaskService) Fail(ctx context.Context, taskID, connectorID string,
return s.taskDAO.FailTask(ctx, taskID, connectorID, message, 1)
}
+// HandleTransientFailure retries a running task until maxRetries is reached.
+func (s *SyncTaskService) HandleTransientFailure(ctx context.Context, taskID, connectorID string, err error, maxRetries int64) (int64, bool, error) {
+ message := ""
+ if err != nil {
+ message = err.Error()
+ }
+ return s.taskDAO.HandleTransientFailure(ctx, taskID, connectorID, message, maxRetries)
+}
+
// CompleteSync commits a successful SYNC task and schedules the next one.
-func (s *SyncTaskService) CompleteSync(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, stats SyncStats) error {
+func (s *SyncTaskService) CompleteSync(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, stats SyncStats) (string, error) {
changed := stats.Added + stats.Updated
return s.taskDAO.CompleteSyncTask(ctx, taskContext, pollRangeEnd, changed, changed, stats.ErrorCount, stats.ErrorMsg)
}
// CompletePrune commits a successful PRUNE task and schedules the next one.
-func (s *SyncTaskService) CompletePrune(ctx context.Context, taskContext SyncTaskContext, removed int64) error {
+func (s *SyncTaskService) CompletePrune(ctx context.Context, taskContext SyncTaskContext, removed int64) (string, error) {
return s.taskDAO.CompletePruneTask(ctx, taskContext, removed)
}
diff --git a/internal/syncer/checkpoint_store.go b/internal/syncer/checkpoint_store.go
new file mode 100644
index 0000000000..018f3c5200
--- /dev/null
+++ b/internal/syncer/checkpoint_store.go
@@ -0,0 +1,91 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ syncerconnector "ragflow/internal/syncer/connector"
+ "sync"
+)
+
+// SyncCheckpointStore persists running sync task checkpoints.
+type SyncCheckpointStore interface {
+ LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error)
+ SaveSyncCheckpoint(ctx context.Context, taskID string, state syncerconnector.SyncCheckpointState) error
+ DeleteSyncCheckpoint(ctx context.Context, taskID string) error
+}
+
+type memorySyncCheckpointStore struct {
+ mu sync.Mutex
+ states map[string]syncerconnector.SyncCheckpointState
+}
+
+func newMemorySyncCheckpointStore() *memorySyncCheckpointStore {
+ return &memorySyncCheckpointStore{states: map[string]syncerconnector.SyncCheckpointState{}}
+}
+
+func cloneSyncCheckpointState(state syncerconnector.SyncCheckpointState) syncerconnector.SyncCheckpointState {
+ clone := state
+ if state.WindowStart != nil {
+ windowStart := *state.WindowStart
+ clone.WindowStart = &windowStart
+ }
+
+ if state.Checkpoint != nil {
+ checkpoint := *state.Checkpoint
+ if checkpoint.UpdatedAt != nil {
+ updatedAt := *checkpoint.UpdatedAt
+ checkpoint.UpdatedAt = &updatedAt
+ }
+ clone.Checkpoint = &checkpoint
+ }
+ return clone
+}
+
+func (s *memorySyncCheckpointStore) LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ state, ok := s.states[taskID]
+ if !ok {
+ return nil, nil
+ }
+ clone := cloneSyncCheckpointState(state)
+ return &clone, nil
+}
+
+func (s *memorySyncCheckpointStore) SaveSyncCheckpoint(ctx context.Context, taskID string, state syncerconnector.SyncCheckpointState) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.states[taskID] = cloneSyncCheckpointState(state)
+ return nil
+}
+
+func (s *memorySyncCheckpointStore) DeleteSyncCheckpoint(ctx context.Context, taskID string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ delete(s.states, taskID)
+ return nil
+}
diff --git a/internal/syncer/config.go b/internal/syncer/config.go
index 2b9604b0e6..a14da4ca7c 100644
--- a/internal/syncer/config.go
+++ b/internal/syncer/config.go
@@ -22,7 +22,6 @@ import (
// Config contains runtime limits for the datasource syncer.
type Config struct {
- PollInterval time.Duration
TaskWorkerCount int
TaskQueueSize int
JobWorkerCount int
@@ -34,7 +33,6 @@ type Config struct {
// DefaultConfig returns the first-version syncer defaults.
func DefaultConfig() Config {
return Config{
- PollInterval: 3 * time.Second,
TaskWorkerCount: 5,
TaskQueueSize: 10,
JobWorkerCount: 400,
@@ -47,10 +45,6 @@ func DefaultConfig() Config {
// Normalize fills unset or invalid config values with defaults.
func (c Config) Normalize() Config {
def := DefaultConfig()
- if c.PollInterval <= 0 {
- c.PollInterval = def.PollInterval
- }
-
if c.TaskWorkerCount <= 0 {
c.TaskWorkerCount = def.TaskWorkerCount
}
diff --git a/internal/syncer/connector/fingerprint.go b/internal/syncer/connector/fingerprint.go
index b463cca42b..6c86506703 100644
--- a/internal/syncer/connector/fingerprint.go
+++ b/internal/syncer/connector/fingerprint.go
@@ -20,15 +20,55 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
+ "path/filepath"
+ "regexp"
+ "strings"
"github.com/zeebo/xxh3"
)
+var syncFilenameUnsafeRE = regexp.MustCompile(`[\\/:*?"<>|]+`)
+
func stableFingerprint(value any) string {
data, err := json.Marshal(value)
if err != nil {
data = []byte(fmt.Sprint(value))
}
+ return contentFingerprint(data)
+}
+
+func contentFingerprint(data []byte) string {
sum := xxh3.Hash128(data).Bytes()
return hex.EncodeToString(sum[:])
}
+
+func syncSourceDocumentFilenameFromDocument(doc SourceDocument) string {
+ name := strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(doc.SemanticIdentifier, "_"))
+ if name == "" {
+ name = strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(doc.SourceID, "_"))
+ }
+ if name == "" {
+ name = "document"
+ }
+
+ extension := strings.TrimSpace(doc.Extension)
+ if extension == "" {
+ extension = ".txt"
+ }
+ if !strings.HasPrefix(extension, ".") {
+ extension = "." + extension
+ }
+ if !strings.HasSuffix(strings.ToLower(name), strings.ToLower(extension)) {
+ name += extension
+ }
+ if len(name) <= 255 {
+ return name
+ }
+
+ ext := filepath.Ext(name)
+ baseLimit := 255 - len(ext)
+ if baseLimit < 1 {
+ return name[:255]
+ }
+ return name[:baseLimit] + ext
+}
diff --git a/internal/syncer/connector/gmail.go b/internal/syncer/connector/gmail.go
index 9f18f6f5c0..1380c5ffcd 100644
--- a/internal/syncer/connector/gmail.go
+++ b/internal/syncer/connector/gmail.go
@@ -100,7 +100,14 @@ func (c *GmailConnector) OpenSync(ctx context.Context, request SyncRequest) (Syn
if !request.FromBeginning && request.WindowStart != nil {
query = gmailTimeRangeQuery(request.WindowStart, request.WindowEnd)
}
- return &gmailSyncSession{connector: c, users: users, batchSize: c.batchSize, query: query}, nil
+ session := &gmailSyncSession{
+ connector: c,
+ users: users,
+ batchSize: c.batchSize,
+ query: query,
+ }
+ session.applyResume(request.Resume)
+ return session, nil
}
// OpenPrune opens one complete Gmail prune snapshot session.
@@ -288,21 +295,28 @@ func (c *GmailConnector) getJSON(ctx context.Context, client *http.Client, apiUR
}
type gmailSyncSession struct {
- connector *GmailConnector
- users []string
- userIndex int
- pageToken string
- batchSize int
- query string
- buffer []SourceDocument
+ connector *GmailConnector
+ users []string
+ userIndex int
+ pageToken string
+ batchSize int
+ query string
+ buffer []gmailBufferedDocument
+ resumePageToken string
+ resumeOffset int
+ resumeFilename string
}
// NextBatch returns the next Gmail document batch.
func (s *gmailSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
documents := make([]SourceDocument, 0, s.batchSize)
+ var checkpoint *SyncCheckpoint
if len(s.buffer) > 0 {
n := min(s.batchSize, len(s.buffer))
- documents = append(documents, s.buffer[:n]...)
+ for _, buffered := range s.buffer[:n] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
s.buffer = s.buffer[n:]
}
for len(documents) < s.batchSize {
@@ -318,13 +332,19 @@ func (s *gmailSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
}
remaining := s.batchSize - len(documents)
if len(batch) > remaining {
- documents = append(documents, batch[:remaining]...)
+ for _, buffered := range batch[:remaining] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
s.buffer = append(s.buffer, batch[remaining:]...)
break
}
- documents = append(documents, batch...)
+ for _, buffered := range batch {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
}
- return SyncBatch{Documents: documents}, nil
+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
}
// Close closes the Gmail sync session.
@@ -333,9 +353,10 @@ func (s *gmailSyncSession) Close() error {
}
// nextDocumentPage fetches one Gmail list page and expands threads.
-func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {
+func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]gmailBufferedDocument, error) {
userEmail := s.users[s.userIndex]
- page, err := s.connector.listThreads(ctx, userEmail, s.query, s.pageToken, gmailItemsPerPage)
+ requestPageToken := s.pageToken
+ page, err := s.connector.listThreads(ctx, userEmail, s.query, requestPageToken, gmailItemsPerPage)
if err != nil {
if isGmailDisabled(err) {
s.advanceUser()
@@ -343,7 +364,8 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocume
}
return nil, err
}
- documents := make([]SourceDocument, 0, len(page.Threads))
+ candidates := make([]gmailBufferedDocument, 0, len(page.Threads))
+ pageOffset := 0
for _, item := range page.Threads {
thread, err := s.connector.loadThread(ctx, userEmail, item.ID)
if err != nil {
@@ -354,9 +376,16 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocume
}
doc, ok := thread.toSourceDocument(userEmail)
if ok {
- documents = append(documents, doc)
+ pageOffset++
+ candidates = append(candidates, gmailBufferedDocument{
+ document: doc,
+ checkpoint: gmailSyncCheckpoint(userEmail, requestPageToken, pageOffset, doc),
+ offset: pageOffset,
+ filename: syncSourceDocumentFilenameFromDocument(doc),
+ })
}
}
+ documents := s.filterResumedDocuments(requestPageToken, candidates)
if page.NextPageToken == "" {
s.advanceUser()
} else {
@@ -365,10 +394,103 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocume
return documents, nil
}
+// applyResume apply resume if resume is available (adjust session's token and pageOffset)
+func (s *gmailSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ if checkpoint == nil || checkpoint.Cursor == "" {
+ return
+ }
+
+ var cursor gmailSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return
+ }
+ if cursor.UserEmail == "" {
+ return
+ }
+ for index, userEmail := range s.users {
+ if userEmail != cursor.UserEmail {
+ continue
+ }
+ s.userIndex = index
+ s.pageToken = cursor.PageToken
+ s.resumePageToken = cursor.PageToken
+ s.resumeFilename = cursor.Filename
+ if cursor.Offset > 0 {
+ s.resumeOffset = cursor.Offset
+ }
+ return
+ }
+}
+
+// filterResumedDocuments applies checkpoint offset using the filename at that offset.
+func (s *gmailSyncSession) filterResumedDocuments(pageToken string, candidates []gmailBufferedDocument) []gmailBufferedDocument {
+ if s.resumeOffset <= 0 {
+ return candidates
+ }
+ if pageToken != s.resumePageToken {
+ s.clearResumeOffset()
+ return candidates
+ }
+ if s.resumeOffset <= len(candidates) && candidates[s.resumeOffset-1].filename == s.resumeFilename {
+ filtered := candidates[:0]
+ for _, candidate := range candidates {
+ if candidate.offset > s.resumeOffset {
+ filtered = append(filtered, candidate)
+ }
+ }
+ s.clearResumeOffset()
+ return filtered
+ }
+
+ filtered := candidates[:0]
+ for _, candidate := range candidates {
+ if candidate.offset >= s.resumeOffset {
+ filtered = append(filtered, candidate)
+ }
+ }
+ s.clearResumeOffset()
+ return filtered
+}
+
+func (s *gmailSyncSession) clearResumeOffset() {
+ s.resumeOffset = 0
+ s.resumePageToken = ""
+ s.resumeFilename = ""
+}
+
// advanceUser moves a Gmail session to the next mailbox.
func (s *gmailSyncSession) advanceUser() {
s.userIndex++
s.pageToken = ""
+ s.clearResumeOffset()
+}
+
+type gmailBufferedDocument struct {
+ document SourceDocument
+ checkpoint *SyncCheckpoint
+ offset int
+ filename string
+}
+
+type gmailSyncCursor struct {
+ UserEmail string `json:"user_email"`
+ PageToken string `json:"page_token,omitempty"`
+ Offset int `json:"offset"`
+ Filename string `json:"filename"`
+}
+
+func gmailSyncCheckpoint(userEmail, pageToken string, offset int, doc SourceDocument) *SyncCheckpoint {
+ filename := syncSourceDocumentFilenameFromDocument(doc)
+ cursor, err := json.Marshal(gmailSyncCursor{UserEmail: userEmail, PageToken: pageToken, Offset: offset, Filename: filename})
+ if err != nil {
+ return nil
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: string(cursor),
+ UpdatedAt: &updatedAt,
+ SourceID: doc.SourceID,
+ }
}
type gmailPruneSession struct {
@@ -523,12 +645,7 @@ func (t gmailThread) toSourceDocument(userEmail string) (SourceDocument, bool) {
UpdatedAt: updatedAt,
SizeBytes: int64(len(blob)),
Metadata: metadata,
- Fingerprint: stableFingerprint(map[string]any{
- "thread_id": t.ID,
- "updated_at": updatedAt,
- "blob": string(blob),
- "metadata": metadata,
- }),
+ Fingerprint: contentFingerprint(blob),
}, true
}
diff --git a/internal/syncer/connector/gmail_test.go b/internal/syncer/connector/gmail_test.go
index 6bfa76f96f..22c3f05e16 100644
--- a/internal/syncer/connector/gmail_test.go
+++ b/internal/syncer/connector/gmail_test.go
@@ -59,6 +59,100 @@ func TestGmailConnectorOpenSync(t *testing.T) {
}
}
+// TestGmailConnectorOpenSyncResumesWithinPage verifies Gmail checkpoint resumes inside a list page.
+func TestGmailConnectorOpenSyncResumesWithinPage(t *testing.T) {
+ connector := newFixtureGmailConnector()
+ connector.GmailConnector.batchSize = 2
+ connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {
+ return gmailThreadListPage{Threads: []struct {
+ ID string `json:"id"`
+ }{{ID: "thread-1"}, {ID: "thread-2"}, {ID: "thread-3"}}}, nil
+ }
+ connector.GmailConnector.getThread = func(ctx context.Context, userEmail, threadID string) (gmailThread, error) {
+ return gmailTestThread(threadID, "Fri, 02 Jan 2026 03:04:05 +0000")
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if len(first.Documents) != 2 {
+ t.Fatalf("first documents len = %d, want 2", len(first.Documents))
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "thread-2" {
+ t.Fatalf("first checkpoint = %+v, want thread-2", first.Checkpoint)
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "thread-3" {
+ t.Fatalf("resume documents = %+v, want thread-3", second.Documents)
+ }
+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "thread-3" {
+ t.Fatalf("resume checkpoint = %+v, want thread-3", second.Checkpoint)
+ }
+}
+
+// TestGmailConnectorResumeUsesFilenameWhenOffsetChanged verifies page deletions resume from the current filename offset.
+func TestGmailConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {
+ connector := newFixtureGmailConnector()
+ connector.GmailConnector.batchSize = 2
+ connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {
+ return gmailThreadListPage{Threads: []struct {
+ ID string `json:"id"`
+ }{{ID: "thread-1"}, {ID: "thread-2"}, {ID: "thread-3"}}}, nil
+ }
+ connector.GmailConnector.getThread = func(ctx context.Context, userEmail, threadID string) (gmailThread, error) {
+ return gmailTestThreadWithSubject(threadID, threadID, "Fri, 02 Jan 2026 03:04:05 +0000")
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "thread-2" {
+ t.Fatalf("first checkpoint = %+v, want thread-2", first.Checkpoint)
+ }
+
+ resumeListCalls := 0
+ connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {
+ resumeListCalls++
+ if pageToken != "" {
+ t.Fatalf("resume used unexpected pageToken=%q", pageToken)
+ }
+ return gmailThreadListPage{Threads: []struct {
+ ID string `json:"id"`
+ }{{ID: "thread-1"}, {ID: "thread-3"}}}, nil
+ }
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "thread-3" {
+ t.Fatalf("resume documents = %+v, want thread-3", second.Documents)
+ }
+ if resumeListCalls != 1 {
+ t.Fatalf("resume list calls = %d, want one checkpoint page", resumeListCalls)
+ }
+}
+
// TestGmailFingerprintStable verifies Gmail fingerprints are stable and content-sensitive.
func TestGmailFingerprintStable(t *testing.T) {
thread := gmailThread{
@@ -91,6 +185,9 @@ func TestGmailFingerprintStable(t *testing.T) {
if doc1.Fingerprint == "" || doc1.Fingerprint != doc2.Fingerprint {
t.Fatalf("fingerprint unstable: %q %q", doc1.Fingerprint, doc2.Fingerprint)
}
+ if doc1.Fingerprint != contentFingerprint(doc1.Blob) {
+ t.Fatalf("fingerprint = %q, want content fingerprint", doc1.Fingerprint)
+ }
changed := thread
changed.Messages = append([]gmailMessage(nil), thread.Messages...)
@@ -191,28 +288,36 @@ func newFixtureGmailConnector() *fixtureGmailConnector {
}{{ID: "thread-1"}}}, nil
}
base.getThread = func(ctx context.Context, userEmail, threadID string) (gmailThread, error) {
- return gmailThread{
- ID: threadID,
- Messages: []gmailMessage{{
- ID: "msg-1",
- Payload: gmailPayload{
- Headers: []gmailHeader{
- {Name: "From", Value: "Alice Example <[email protected]>"},
- {Name: "To", Value: "Bob <[email protected]>"},
- {Name: "Subject", Value: "Hello/World"},
- {Name: "Date", Value: "Fri, 02 Jan 2026 03:04:05 +0000"},
- },
- Parts: []gmailPart{{
- MimeType: "text/plain",
- Body: gmailBody{Data: base64.RawURLEncoding.EncodeToString([]byte("Body text"))},
- }},
- },
- }},
- }, nil
+ return gmailTestThread(threadID, "Fri, 02 Jan 2026 03:04:05 +0000")
}
return connector
}
+func gmailTestThread(threadID, date string) (gmailThread, error) {
+ return gmailTestThreadWithSubject(threadID, "Hello/World", date)
+}
+
+func gmailTestThreadWithSubject(threadID, subject, date string) (gmailThread, error) {
+ return gmailThread{
+ ID: threadID,
+ Messages: []gmailMessage{{
+ ID: "msg-1",
+ Payload: gmailPayload{
+ Headers: []gmailHeader{
+ {Name: "From", Value: "Alice Example <[email protected]>"},
+ {Name: "To", Value: "Bob <[email protected]>"},
+ {Name: "Subject", Value: subject},
+ {Name: "Date", Value: date},
+ },
+ Parts: []gmailPart{{
+ MimeType: "text/plain",
+ Body: gmailBody{Data: base64.RawURLEncoding.EncodeToString([]byte("Body text"))},
+ }},
+ },
+ }},
+ }, nil
+}
+
// TestGmailTimeRangeQuery verifies Python-compatible after and before seconds.
func TestGmailTimeRangeQuery(t *testing.T) {
start := time.Unix(100, 0).UTC()
diff --git a/internal/syncer/connector/google_drive.go b/internal/syncer/connector/google_drive.go
index 96da7d9113..cef4f821b6 100644
--- a/internal/syncer/connector/google_drive.go
+++ b/internal/syncer/connector/google_drive.go
@@ -38,7 +38,7 @@ import (
const (
defaultGoogleDriveBatchSize = 32
- defaultGoogleDriveSizeThreshold = 10 * 1024 * 1024
+ defaultGoogleDriveSizeThreshold = 64 * 1024 * 1024
googleDriveItemsPerPage = 100
googleDriveRequestTimeout = 60 * time.Second
googleDriveOAuthTokenURL = "https://oauth2.googleapis.com/token"
@@ -144,14 +144,15 @@ func (c *GoogleDriveConnector) OpenSync(ctx context.Context, request SyncRequest
if err != nil {
return nil, err
}
- return &googleDriveSyncSession{
+ session := &googleDriveSyncSession{
connector: c,
scopes: scopes,
scopeIndex: 0,
batchSize: c.batchSize,
windowStart: request.WindowStart,
windowEnd: request.WindowEnd,
- }, nil
+ }
+ return session.applyResume(request.Resume), nil
}
// OpenPrune opens one complete Google Drive prune snapshot session.
@@ -540,16 +541,19 @@ func (c *GoogleDriveConnector) getBytes(ctx context.Context, client *http.Client
}
type googleDriveSyncSession struct {
- connector *GoogleDriveConnector
- scopes []googleDriveScope
- scopeIndex int
- pageToken string
- batchSize int
- windowStart *time.Time
- windowEnd time.Time
- buffer []SourceDocument
- seen map[string]struct{}
- folderSeen map[string]struct{}
+ connector *GoogleDriveConnector
+ scopes []googleDriveScope
+ scopeIndex int
+ pageToken string
+ batchSize int
+ windowStart *time.Time
+ windowEnd time.Time
+ buffer []googleDriveBufferedDocument
+ seen map[string]struct{}
+ folderSeen map[string]struct{}
+ resumePageToken string
+ resumePageOffset int
+ resumeFilename string
}
// NextBatch returns the next Google Drive document batch.
@@ -561,9 +565,13 @@ func (s *googleDriveSyncSession) NextBatch(ctx context.Context) (SyncBatch, erro
s.folderSeen = map[string]struct{}{}
}
documents := make([]SourceDocument, 0, s.batchSize)
+ var checkpoint *SyncCheckpoint
if len(s.buffer) > 0 {
n := min(s.batchSize, len(s.buffer))
- documents = append(documents, s.buffer[:n]...)
+ for _, buffered := range s.buffer[:n] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
s.buffer = s.buffer[n:]
}
for len(documents) < s.batchSize {
@@ -579,18 +587,25 @@ func (s *googleDriveSyncSession) NextBatch(ctx context.Context) (SyncBatch, erro
}
remaining := s.batchSize - len(documents)
if len(batch) > remaining {
- documents = append(documents, batch[:remaining]...)
+ for _, buffered := range batch[:remaining] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
s.buffer = append(s.buffer, batch[remaining:]...)
break
}
- documents = append(documents, batch...)
+ for _, buffered := range batch {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
}
- return SyncBatch{Documents: documents}, nil
+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
}
-func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {
+func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]googleDriveBufferedDocument, error) {
scope := s.scopes[s.scopeIndex]
- page, err := s.connector.listFilePageWithRetry(ctx, scope, s.pageToken, s.windowStart, s.windowEnd)
+ requestPageToken := s.pageToken
+ page, err := s.connector.listFilePageWithRetry(ctx, scope, requestPageToken, s.windowStart, s.windowEnd)
if err != nil {
if isGooglePermissionDeniedOrNotFound(err) {
s.advanceScope()
@@ -598,18 +613,22 @@ func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]Source
}
return nil, err
}
- docs := make([]SourceDocument, 0, len(page.Files))
+ candidates := make([]googleDriveBufferedDocument, 0, len(page.Files))
+ pageOffset := 0
for _, file := range page.Files {
doc, ok := file.toSourceDocument(scope.userEmail, s.connector.allowImages)
if !ok {
continue
}
- if _, ok = s.seen[doc.SourceID]; ok {
- continue
- }
- s.seen[doc.SourceID] = struct{}{}
- docs = append(docs, doc)
- }
+ pageOffset++
+ candidates = append(candidates, googleDriveBufferedDocument{
+ document: doc,
+ checkpoint: s.syncCheckpoint(scope, requestPageToken, pageOffset, doc),
+ offset: pageOffset,
+ filename: syncSourceDocumentFilenameFromDocument(doc),
+ })
+ }
+ docs := s.filterSeenDocuments(s.filterResumedDocuments(requestPageToken, candidates))
if page.NextPageToken == "" {
if err = s.finishScope(ctx, scope); err != nil {
return nil, err
@@ -643,6 +662,7 @@ func (s *googleDriveSyncSession) finishScope(ctx context.Context, scope googleDr
func (s *googleDriveSyncSession) advanceScope() {
s.scopeIndex++
s.pageToken = ""
+ s.clearResumeOffset()
}
// Close closes the Google Drive sync session.
@@ -655,6 +675,185 @@ func (s *googleDriveSyncSession) Fetch(ctx context.Context, ref FetchReference)
return s.connector.Fetch(ctx, ref)
}
+func (s *googleDriveSyncSession) filterSeenDocuments(candidates []googleDriveBufferedDocument) []googleDriveBufferedDocument {
+ filtered := candidates[:0]
+ for _, candidate := range candidates {
+ if _, ok := s.seen[candidate.document.SourceID]; ok {
+ continue
+ }
+ s.seen[candidate.document.SourceID] = struct{}{}
+ filtered = append(filtered, candidate)
+ }
+ return filtered
+}
+
+// applyResume apply resume if checkpoint is available
+func (s *googleDriveSyncSession) applyResume(checkpoint *SyncCheckpoint) *googleDriveSyncSession {
+ if checkpoint == nil || checkpoint.Cursor == "" {
+ return s
+ }
+ var cursor googleDriveSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return s
+ }
+ if len(cursor.Scopes) > 0 && cursor.ScopeIndex >= 0 && cursor.ScopeIndex < len(cursor.Scopes) {
+ scopes := make([]googleDriveScope, 0, len(cursor.Scopes))
+ for _, scope := range cursor.Scopes {
+ scopes = append(scopes, scope.toScope())
+ }
+ s.scopes = scopes
+ s.scopeIndex = cursor.ScopeIndex
+ s.markFolderScopesSeen()
+ } else if scope, ok := cursor.Scope.toScopeIfValid(); ok {
+ if index := googleDriveScopeIndex(s.scopes, scope); index >= 0 {
+ s.scopeIndex = index
+ } else {
+ s.scopes = append(s.scopes, scope)
+ s.scopeIndex = len(s.scopes) - 1
+ }
+ }
+ s.pageToken = cursor.PageToken
+ s.resumePageToken = cursor.PageToken
+ s.resumeFilename = cursor.Filename
+ if cursor.Offset > 0 {
+ s.resumePageOffset = cursor.Offset
+ }
+ return s
+}
+
+func (s *googleDriveSyncSession) filterResumedDocuments(pageToken string, candidates []googleDriveBufferedDocument) []googleDriveBufferedDocument {
+ if s.resumePageOffset <= 0 {
+ return candidates
+ }
+ if pageToken != s.resumePageToken {
+ s.clearResumeOffset()
+ return candidates
+ }
+ if s.resumePageOffset <= len(candidates) && candidates[s.resumePageOffset-1].filename == s.resumeFilename {
+ filtered := candidates[:0]
+ for _, candidate := range candidates {
+ if candidate.offset > s.resumePageOffset {
+ filtered = append(filtered, candidate)
+ }
+ }
+ s.clearResumeOffset()
+ return filtered
+ }
+
+ filtered := candidates[:0]
+ for _, candidate := range candidates {
+ if candidate.offset >= s.resumePageOffset {
+ filtered = append(filtered, candidate)
+ }
+ }
+ s.clearResumeOffset()
+ return filtered
+}
+
+func (s *googleDriveSyncSession) clearResumeOffset() {
+ s.resumePageOffset = 0
+ s.resumePageToken = ""
+ s.resumeFilename = ""
+}
+
+func (s *googleDriveSyncSession) syncCheckpoint(scope googleDriveScope, pageToken string, offset int, doc SourceDocument) *SyncCheckpoint {
+ scopes := make([]googleDriveScopeCursor, 0, len(s.scopes))
+ for _, scope := range s.scopes {
+ scopes = append(scopes, googleDriveScopeCursorFromScope(scope))
+ }
+ cursor, err := json.Marshal(googleDriveSyncCursor{
+ Scopes: scopes,
+ Scope: googleDriveScopeCursorFromScope(scope),
+ ScopeIndex: s.scopeIndex,
+ PageToken: pageToken,
+ Offset: offset,
+ Filename: syncSourceDocumentFilenameFromDocument(doc),
+ })
+ if err != nil {
+ return nil
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: string(cursor),
+ UpdatedAt: &updatedAt,
+ SourceID: doc.SourceID,
+ }
+}
+
+func (s *googleDriveSyncSession) markFolderScopesSeen() {
+ if s.folderSeen == nil {
+ s.folderSeen = map[string]struct{}{}
+ }
+ for _, scope := range s.scopes {
+ if scope.corpora == "folder" && scope.folderID != "" {
+ s.folderSeen[scope.folderID] = struct{}{}
+ }
+ }
+}
+
+type googleDriveBufferedDocument struct {
+ document SourceDocument
+ checkpoint *SyncCheckpoint
+ offset int
+ filename string
+}
+
+type googleDriveSyncCursor struct {
+ Scopes []googleDriveScopeCursor `json:"scopes,omitempty"`
+ Scope googleDriveScopeCursor `json:"scope"`
+ ScopeIndex int `json:"scope_index"`
+ PageToken string `json:"page_token,omitempty"`
+ Offset int `json:"offset"`
+ Filename string `json:"filename"`
+}
+
+type googleDriveScopeCursor struct {
+ UserEmail string `json:"user_email"`
+ Corpora string `json:"corpora"`
+ DriveID string `json:"drive_id,omitempty"`
+ FolderID string `json:"folder_id,omitempty"`
+ IncludeSharedWithMe bool `json:"include_shared_with_me,omitempty"`
+ SharedWithMeOnly bool `json:"shared_with_me_only,omitempty"`
+}
+
+func googleDriveScopeCursorFromScope(scope googleDriveScope) googleDriveScopeCursor {
+ return googleDriveScopeCursor{
+ UserEmail: scope.userEmail,
+ Corpora: scope.corpora,
+ DriveID: scope.driveID,
+ FolderID: scope.folderID,
+ IncludeSharedWithMe: scope.includeSharedWithMe,
+ SharedWithMeOnly: scope.sharedWithMeOnly,
+ }
+}
+
+func (c googleDriveScopeCursor) toScope() googleDriveScope {
+ return googleDriveScope{
+ userEmail: c.UserEmail,
+ corpora: c.Corpora,
+ driveID: c.DriveID,
+ folderID: c.FolderID,
+ includeSharedWithMe: c.IncludeSharedWithMe,
+ sharedWithMeOnly: c.SharedWithMeOnly,
+ }
+}
+
+func (c googleDriveScopeCursor) toScopeIfValid() (googleDriveScope, bool) {
+ if c.UserEmail == "" || c.Corpora == "" {
+ return googleDriveScope{}, false
+ }
+ return c.toScope(), true
+}
+
+func googleDriveScopeIndex(scopes []googleDriveScope, target googleDriveScope) int {
+ for index, scope := range scopes {
+ if scope == target {
+ return index
+ }
+ }
+ return -1
+}
+
type googleDrivePruneSession struct {
connector *GoogleDriveConnector
scopes []googleDriveScope
diff --git a/internal/syncer/connector/google_drive_test.go b/internal/syncer/connector/google_drive_test.go
index 045188938f..5673caf66e 100644
--- a/internal/syncer/connector/google_drive_test.go
+++ b/internal/syncer/connector/google_drive_test.go
@@ -92,6 +92,125 @@ func TestGoogleDriveConnectorOpenSyncUsesWindowFingerprintAndFetch(t *testing.T)
}
}
+// TestGoogleDriveConnectorOpenSyncResumesWithinPage verifies Drive checkpoints resume inside a list page.
+func TestGoogleDriveConnectorOpenSyncResumesWithinPage(t *testing.T) {
+ connector, err := NewGoogleDriveConnector(map[string]any{
+ "my_drive_emails": "[email protected]",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "google_primary_admin": "[email protected]",
+ "google_tokens": `{"client_id":"client","client_secret":"secret","refresh_token":"refresh"}`,
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewGoogleDriveConnector failed: %v", err)
+ }
+ files := []googleDriveFile{
+ googleDriveTestFile("file-1", "One.txt", "2026-01-01T00:00:00Z"),
+ googleDriveTestFile("file-2", "Two.txt", "2026-01-02T00:00:00Z"),
+ googleDriveTestFile("file-3", "Three.txt", "2026-01-03T00:00:00Z"),
+ }
+ var pageTokens []string
+ connector.listFiles = func(ctx context.Context, userEmail string, request googleDriveListRequest) (googleDriveFilePage, error) {
+ pageTokens = append(pageTokens, request.PageToken)
+ return googleDriveFilePage{Files: files}, nil
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if len(first.Documents) != 2 || first.Documents[0].SourceID != "https://drive.google.com/file/d/file-1" || first.Documents[1].SourceID != "https://drive.google.com/file/d/file-2" {
+ t.Fatalf("first documents = %+v", first.Documents)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "https://drive.google.com/file/d/file-2" {
+ t.Fatalf("first checkpoint = %+v, want file-2", first.Checkpoint)
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://drive.google.com/file/d/file-3" {
+ t.Fatalf("resume documents = %+v, want file-3", second.Documents)
+ }
+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "https://drive.google.com/file/d/file-3" {
+ t.Fatalf("resume checkpoint = %+v, want file-3", second.Checkpoint)
+ }
+ if len(pageTokens) != 2 || pageTokens[1] != "" {
+ t.Fatalf("page tokens = %v, want resume from first page", pageTokens)
+ }
+}
+
+// TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged verifies page deletions resume from the current filename offset.
+func TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {
+ connector, err := NewGoogleDriveConnector(map[string]any{
+ "my_drive_emails": "[email protected]",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "google_primary_admin": "[email protected]",
+ "google_tokens": `{"client_id":"client","client_secret":"secret","refresh_token":"refresh"}`,
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewGoogleDriveConnector failed: %v", err)
+ }
+ files := []googleDriveFile{
+ googleDriveTestFile("file-1", "One.txt", "2026-01-01T00:00:00Z"),
+ googleDriveTestFile("file-2", "Two.txt", "2026-01-02T00:00:00Z"),
+ googleDriveTestFile("file-3", "Three.txt", "2026-01-03T00:00:00Z"),
+ }
+ connector.listFiles = func(ctx context.Context, userEmail string, request googleDriveListRequest) (googleDriveFilePage, error) {
+ return googleDriveFilePage{Files: files}, nil
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "https://drive.google.com/file/d/file-2" {
+ t.Fatalf("first checkpoint = %+v, want file-2", first.Checkpoint)
+ }
+
+ files = []googleDriveFile{
+ googleDriveTestFile("file-1", "One.txt", "2026-01-01T00:00:00Z"),
+ googleDriveTestFile("file-3", "Three.txt", "2026-01-03T00:00:00Z"),
+ }
+ resumeListCalls := 0
+ connector.listFiles = func(ctx context.Context, userEmail string, request googleDriveListRequest) (googleDriveFilePage, error) {
+ resumeListCalls++
+ if request.PageToken != "" {
+ t.Fatalf("resume used unexpected pageToken=%q", request.PageToken)
+ }
+ return googleDriveFilePage{Files: files}, nil
+ }
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://drive.google.com/file/d/file-3" {
+ t.Fatalf("resume documents = %+v, want file-3", second.Documents)
+ }
+ if resumeListCalls != 1 {
+ t.Fatalf("resume list calls = %d, want one checkpoint page", resumeListCalls)
+ }
+}
+
// TestGoogleDriveSharedFolderScopesRecurse verifies shared folders walk child folders.
func TestGoogleDriveSharedFolderScopesRecurse(t *testing.T) {
connector, err := NewGoogleDriveConnector(map[string]any{
@@ -233,3 +352,13 @@ func TestGoogleDriveFileQueryUsesIncrementalWindow(t *testing.T) {
t.Fatalf("query = %q", query)
}
}
+
+func googleDriveTestFile(id, name, modifiedTime string) googleDriveFile {
+ return googleDriveFile{
+ ID: id,
+ Name: name,
+ MimeType: "text/plain",
+ ModifiedTime: modifiedTime,
+ WebViewLink: "https://drive.google.com/file/d/" + id + "/view",
+ }
+}
diff --git a/internal/syncer/connector/mock/mock.go b/internal/syncer/connector/mock/mock.go
index 7a1e3c8bad..efe26780eb 100644
--- a/internal/syncer/connector/mock/mock.go
+++ b/internal/syncer/connector/mock/mock.go
@@ -32,6 +32,7 @@ type Connector struct {
FetchBlobs map[string][]byte
OnSyncBatch func(index int)
OnPruneBatch func(index int)
+ SyncRequests []syncerconnector.SyncRequest
}
// Validate returns the configured validation error.
@@ -41,6 +42,7 @@ func (c *Connector) Validate(ctx context.Context) error {
// OpenSync opens a mock sync session.
func (c *Connector) OpenSync(ctx context.Context, request syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {
+ c.SyncRequests = append(c.SyncRequests, request)
return &SyncSession{connector: c}, nil
}
diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.go
index 4454b0f866..f064a40871 100644
--- a/internal/syncer/connector/models.go
+++ b/internal/syncer/connector/models.go
@@ -37,19 +37,48 @@ type FetchReference struct {
SizeHint int64
}
+// SyncCheckpoint is a connector-owned resume point.
+type SyncCheckpoint struct {
+ Cursor string `json:"cursor,omitempty"`
+ UpdatedAt *time.Time `json:"updated_at,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
+}
+
+// SyncCheckpointState is the running checkpoint for one sync task.
+type SyncCheckpointState struct {
+ Version int `json:"version"`
+ TaskID string `json:"task_id"`
+ ConnectorID string `json:"connector_id"`
+ KBID string `json:"kb_id"`
+ WindowStart *time.Time `json:"window_start,omitempty"`
+ WindowEnd time.Time `json:"window_end"`
+ NextCommitSeq int64 `json:"next_commit_seq"`
+ Checkpoint *SyncCheckpoint `json:"checkpoint,omitempty"`
+ Added int64 `json:"added,omitempty"`
+ Updated int64 `json:"updated,omitempty"`
+ Skipped int64 `json:"skipped,omitempty"`
+ ErrorCount int64 `json:"error_count,omitempty"`
+ ErrorMsg string `json:"error_msg,omitempty"`
+}
+
// SyncRequest describes one fixed sync window.
type SyncRequest struct {
- TaskID string
- ConnectorID string
- KBID string
+ TaskID string
+ ConnectorID string
+ KBID string
+ SourceType string
+ Fingerprints map[string]string
+
FromBeginning bool
WindowStart *time.Time
WindowEnd time.Time
+ Resume *SyncCheckpoint
}
// SyncBatch contains one serially processed batch.
type SyncBatch struct {
- Documents []SourceDocument
+ Documents []SourceDocument
+ Checkpoint *SyncCheckpoint
}
// PruneRequest describes one complete prune snapshot request.
diff --git a/internal/syncer/job_executor.go b/internal/syncer/job_executor.go
index e6f7c8f6fb..2b4c67c1ab 100644
--- a/internal/syncer/job_executor.go
+++ b/internal/syncer/job_executor.go
@@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
"sync"
"time"
)
@@ -53,14 +54,16 @@ type syncJobFunc func(context.Context) (service.SyncStats, error)
// syncJobResult the stats
type syncJobResult struct {
- stats service.SyncStats
- err error
+ stats service.SyncStats
+ checkpoint *syncerconnector.SyncCheckpoint
+ err error
}
type syncJob struct {
- ctx context.Context
- fn syncJobFunc
- done chan syncJobResult
+ ctx context.Context
+ fn syncJobFunc
+ checkpoint *syncerconnector.SyncCheckpoint
+ done chan syncJobResult
}
// SyncJobQueue is one Coordinator-owned queue feeding the fair dispatcher.
@@ -73,12 +76,16 @@ type SyncJobQueue struct {
// Submit adds one BatchJob to this task's dispatcher queue.
func (q *SyncJobQueue) Submit(ctx context.Context, fn syncJobFunc) (<-chan syncJobResult, error) {
+ return q.submit(ctx, fn, nil)
+}
+
+func (q *SyncJobQueue) submit(ctx context.Context, fn syncJobFunc, checkpoint *syncerconnector.SyncCheckpoint) (<-chan syncJobResult, error) {
if q == nil {
return nil, fmt.Errorf("sync job queue is nil")
}
done := make(chan syncJobResult, 1) // done channel
- job := &syncJob{ctx: ctx, fn: fn, done: done}
+ job := &syncJob{ctx: ctx, fn: fn, checkpoint: checkpoint, done: done}
select {
case <-ctx.Done():
return nil, ctx.Err()
@@ -335,6 +342,6 @@ func (e *SyncJobExecutor) work() {
continue
}
stats, err := job.fn(job.ctx) // run the job
- job.done <- syncJobResult{stats: stats, err: err}
+ job.done <- syncJobResult{stats: stats, checkpoint: job.checkpoint, err: err}
}
}
diff --git a/internal/syncer/prune_runner.go b/internal/syncer/prune_runner.go
index 219d42e057..8da242c6d3 100644
--- a/internal/syncer/prune_runner.go
+++ b/internal/syncer/prune_runner.go
@@ -36,9 +36,9 @@ func NewPruneRunner(taskService *service.SyncTaskService, pruneService *service.
}
// Run collects the full prune snapshot before deleting stale documents.
-func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {
+func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) (string, error) {
if r.pruneService == nil {
- return errors.New("prune service is not configured")
+ return "", errors.New("prune service is not configured")
}
// Get the run session
session, err := connector.OpenPrune(ctx, syncerconnector.PruneRequest{
@@ -47,21 +47,21 @@ func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskConte
KBID: taskContext.Knowledgebase.ID,
})
if err != nil {
- return err
+ return "", err
}
defer session.Close()
retain := map[string]struct{}{}
for {
if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {
- return err
+ return "", err
}
batch, nextErr := session.NextBatch(ctx)
if errors.Is(nextErr, io.EOF) {
break
}
if nextErr != nil {
- return nextErr
+ return "", nextErr
}
for _, doc := range batch.Documents {
service.AddRetainDocumentID(retain, taskContext.Knowledgebase.ID, taskContext.Connector.ID, doc.SourceID)
@@ -69,11 +69,11 @@ func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskConte
}
if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {
- return err
+ return "", err
}
removed, err := r.pruneService.DeleteStale(ctx, taskContext, retain)
if err != nil {
- return err
+ return "", err
}
return r.taskService.CompletePrune(ctx, taskContext, removed)
}
diff --git a/internal/syncer/scheduler.go b/internal/syncer/scheduler.go
index f51da67ec5..fe17e2954c 100644
--- a/internal/syncer/scheduler.go
+++ b/internal/syncer/scheduler.go
@@ -20,8 +20,12 @@ import (
"context"
"errors"
"ragflow/internal/common"
+ "ragflow/internal/entity"
"ragflow/internal/service"
+ "sync"
"time"
+
+ "go.uber.org/zap"
)
// TaskEnvelope is the only payload sent through the task queue.
@@ -31,34 +35,36 @@ type TaskEnvelope struct {
stopHeartbeat func()
}
-// SyncTaskBroker publishes and pulls syncer task wake-up messages.
+// SyncTaskBroker publishes and subscribes to syncer task wake-up messages.
type SyncTaskBroker interface {
InitSyncerStream() error
InitSyncerConsumer() error
PublishSyncerTask(taskID string) error
- FetchSyncerTasks(batchSize int) ([]common.TaskHandle, error)
+ PublishSyncerTaskWakeup(taskID string) error
+ SubscribeSyncerTasks(ctx context.Context, handler func(common.TaskHandle)) error
}
// Scheduler discovers due work and enqueues task IDs for workers.
type Scheduler struct {
- pollInterval time.Duration
- queue chan<- TaskEnvelope
- taskService *service.SyncTaskService
- broker SyncTaskBroker
+ queue chan<- TaskEnvelope
+ taskService *service.SyncTaskService
+ broker SyncTaskBroker
+ timerMu sync.Mutex
+ timers map[string]*time.Timer
}
// NewScheduler creates a global scheduler for datasource sync tasks.
-func NewScheduler(pollInterval time.Duration, queue chan<- TaskEnvelope, taskService *service.SyncTaskService) *Scheduler {
- return &Scheduler{pollInterval: pollInterval, queue: queue, taskService: taskService}
+func NewScheduler(queue chan<- TaskEnvelope, taskService *service.SyncTaskService) *Scheduler {
+ return &Scheduler{queue: queue, taskService: taskService, timers: map[string]*time.Timer{}}
}
// NewNATSScheduler creates a JetStream-driven scheduler with DB reconciliation.
-func NewNATSScheduler(pollInterval time.Duration, queue chan<- TaskEnvelope, taskService *service.SyncTaskService, broker SyncTaskBroker) *Scheduler {
+func NewNATSScheduler(queue chan<- TaskEnvelope, taskService *service.SyncTaskService, broker SyncTaskBroker) *Scheduler {
return &Scheduler{
- pollInterval: pollInterval,
- queue: queue,
- taskService: taskService,
- broker: broker,
+ queue: queue,
+ taskService: taskService,
+ broker: broker,
+ timers: map[string]*time.Timer{},
}
}
@@ -77,127 +83,162 @@ func (s *Scheduler) runNATS(ctx context.Context) error {
if err := s.broker.InitSyncerConsumer(); err != nil {
return err
}
+ if err := s.broker.SubscribeSyncerTasks(ctx, func(handle common.TaskHandle) {
+ if err := s.enqueueHandle(ctx, handle); err != nil && ctx.Err() == nil {
+ common.Error("syncer scheduler enqueue failed", err)
+ }
+ }); err != nil {
+ return err
+ }
// scan DB for first time run
if err := s.publishStartupTasks(ctx); err != nil && ctx.Err() == nil {
common.Error("syncer scheduler startup publish failed", err)
}
- ticker := time.NewTicker(s.pollInterval)
- defer ticker.Stop()
- for {
- if err := ctx.Err(); err != nil {
- return err
- }
-
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-ticker.C: // Run `publishDueTasks` periodically
- if err := s.publishDueTasks(ctx); err != nil && ctx.Err() == nil {
- common.Error("syncer scheduler due publish failed", err)
- }
- continue
- default:
- }
-
- // check `scheduler`'s task queue's slot
- available := s.queueAvailable()
- if available <= 0 {
- if err := waitNATSFetchCapacity(ctx); err != nil {
- return err
- }
- continue
- }
+ <-ctx.Done()
+ s.stopTimers()
+ return ctx.Err()
+}
- // pull `available` tasks from nats
- handles, err := s.broker.FetchSyncerTasks(available)
- if err != nil {
- common.Error("syncer scheduler fetch failed", err)
- if waitErr := waitNATSFetchCapacity(ctx); waitErr != nil {
- return waitErr
- }
- } else if len(handles) == 0 {
- if waitErr := waitNATSFetchCapacity(ctx); waitErr != nil {
- return waitErr
- }
- } else if err = s.enqueueHandles(ctx, handles); err != nil { // put tasks to task queue
- return err
- }
+func (s *Scheduler) enqueueHandle(ctx context.Context, handle common.TaskHandle) error {
+ message := handle.GetMessage()
+ if message.TaskID == "" {
+ _ = handle.Ack()
+ return nil
}
-}
-func waitNATSFetchCapacity(ctx context.Context) error {
+ stopHeartbeat := startHandleHeartbeat(ctx, handle)
select {
case <-ctx.Done():
+ stopHeartbeat()
+ _ = handle.Nack()
return ctx.Err()
- case <-time.After(100 * time.Millisecond):
+ case s.queue <- TaskEnvelope{TaskID: message.TaskID, Handle: handle, stopHeartbeat: stopHeartbeat}:
return nil
}
}
-// enqueueHandles put task to `scheduler`'s task queue
-func (s *Scheduler) enqueueHandles(ctx context.Context, handles []common.TaskHandle) error {
- for index, handle := range handles {
- message := handle.GetMessage()
- if message.TaskID == "" {
- _ = handle.Ack()
- continue
- }
-
- stopHeartbeat := startHandleHeartbeat(ctx, handle)
- select {
- case <-ctx.Done():
- stopHeartbeat()
- for _, pending := range handles[index:] {
- _ = pending.Nack()
- }
- return ctx.Err()
- case s.queue <- TaskEnvelope{TaskID: message.TaskID, Handle: handle, stopHeartbeat: stopHeartbeat}:
- }
- }
- return nil
-}
-
func (s *Scheduler) queueAvailable() int {
return cap(s.queue) - len(s.queue)
}
-// publishStartupTasks scan DB for first time run
+// publishStartupTasks scans DB once for startup reconciliation.
func (s *Scheduler) publishStartupTasks(ctx context.Context) error {
// TODO restores running tasks during syncer startup.
if err := s.taskService.RecoverRunning(ctx); err != nil {
return err
}
- tasks, err := s.taskService.ListStartupTasks(ctx)
+ tasks, err := s.taskService.ListScheduledTasks(ctx)
if err != nil {
return err
}
- // publish task to nats
- var publishErr error
for _, task := range tasks {
- if err = s.broker.PublishSyncerTask(task.ID); err != nil {
- publishErr = errors.Join(publishErr, err)
+ if err = s.ScheduleTask(ctx, task); err != nil {
+ return err
}
}
- return publishErr
+ return nil
}
-// publishDueTasks publishes due scheduled DB tasks to nats
-func (s *Scheduler) publishDueTasks(ctx context.Context) error {
- now := time.Now()
- tasks, err := s.taskService.ListDueTasks(ctx, now)
- if err != nil {
+// ScheduleTask publishes a due scheduled task or arms a one-shot timer.
+func (s *Scheduler) ScheduleTask(ctx context.Context, task service.ScheduledSyncTask) error {
+ delay, schedule := s.taskDelay(task, time.Now())
+ if !schedule {
+ return nil
+ }
+ return s.ScheduleTaskAfter(ctx, task.ID, delay)
+}
+
+// ScheduleTaskAfter publishes a task after delay.
+func (s *Scheduler) ScheduleTaskAfter(ctx context.Context, taskID string, delay time.Duration) error {
+ if s == nil || s.broker == nil || taskID == "" {
+ return nil
+ }
+ if delay <= 0 {
+ return s.publishTask(ctx, taskID)
+ }
+ s.timerMu.Lock()
+ if existing := s.timers[taskID]; existing != nil {
+ existing.Stop()
+ }
+ timer := time.AfterFunc(delay, func() {
+ if err := s.publishTaskWakeup(ctx, taskID); err != nil && ctx.Err() == nil {
+ common.Warn("syncer scheduler timer publish failed", zap.String("task_id", taskID), zap.Error(err))
+ _ = s.ScheduleTaskAfter(ctx, taskID, 3*time.Second)
+ return
+ }
+ s.timerMu.Lock()
+ delete(s.timers, taskID)
+ s.timerMu.Unlock()
+ })
+ s.timers[taskID] = timer
+ s.timerMu.Unlock()
+ return nil
+}
+
+func (s *Scheduler) publishTaskWakeup(ctx context.Context, taskID string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if err := s.broker.PublishSyncerTaskWakeup(taskID); err != nil {
+ common.Warn("syncer task wakeup publish failed", zap.String("task_id", taskID), zap.Error(err))
return err
}
+ return nil
+}
- var publishErr error
- for _, task := range tasks {
- if err = s.broker.PublishSyncerTask(task.ID); err != nil {
- publishErr = errors.Join(publishErr, err)
+func (s *Scheduler) publishTask(ctx context.Context, taskID string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if err := s.broker.PublishSyncerTask(taskID); err != nil {
+ common.Warn("syncer task publish failed", zap.String("task_id", taskID), zap.Error(err))
+ return err
+ }
+ return nil
+}
+
+func (s *Scheduler) stopTimers() {
+ s.timerMu.Lock()
+ defer s.timerMu.Unlock()
+ for taskID, timer := range s.timers {
+ timer.Stop()
+ delete(s.timers, taskID)
+ }
+}
+
+// taskDelay reports the delay before publication and whether the task must be scheduled at all.
+func (s *Scheduler) taskDelay(task service.ScheduledSyncTask, now time.Time) (time.Duration, bool) {
+ freq := int64(0)
+ switch task.TaskType {
+ case service.TaskTypeSync:
+ freq = task.ConnectorRefreshFreq
+ case service.TaskTypePrune:
+ if !syncerConfigBool(task.ConnectorConfig, "sync_deleted_files") {
+ return 0, false
}
+ freq = task.ConnectorPruneFreq
+ }
+ if freq <= 0 || task.UpdateDate == nil {
+ return 0, true
+ }
+ return task.UpdateDate.Add(time.Duration(freq) * time.Minute).Sub(now), true
+}
+
+func syncerConfigBool(config entity.JSONMap, key string) bool {
+ value, ok := config[key]
+ if !ok {
+ return false
+ }
+ switch typed := value.(type) {
+ case bool:
+ return typed
+ case string:
+ return typed == "1" || typed == "true" || typed == "TRUE"
+ default:
+ return false
}
- return publishErr
}
diff --git a/internal/syncer/sync_runner.go b/internal/syncer/sync_runner.go
index e2b7696765..279d68fcf1 100644
--- a/internal/syncer/sync_runner.go
+++ b/internal/syncer/sync_runner.go
@@ -22,9 +22,12 @@ import (
"fmt"
"io"
"math/rand"
+ "ragflow/internal/common"
"ragflow/internal/service"
syncerconnector "ragflow/internal/syncer/connector"
"time"
+
+ "go.uber.org/zap"
)
var errSyncTaskCanceled = errors.New("sync task canceled")
@@ -38,49 +41,61 @@ type SyncRunner struct {
sink service.DocumentSink
idResolver *service.DocumentIDResolver
queue *SyncJobQueue
+ checkpoints SyncCheckpointStore
}
// NewSyncRunner creates a SYNC runner.
-func NewSyncRunner(config TaskCoordinatorConfig, taskService *service.SyncTaskService, sink service.DocumentSink, idResolver *service.DocumentIDResolver, queue *SyncJobQueue) *SyncRunner {
- return &SyncRunner{config: config, taskService: taskService, sink: sink, idResolver: idResolver, queue: queue}
+func NewSyncRunner(config TaskCoordinatorConfig, taskService *service.SyncTaskService, sink service.DocumentSink, idResolver *service.DocumentIDResolver, queue *SyncJobQueue, checkpoints SyncCheckpointStore) *SyncRunner {
+ if checkpoints == nil {
+ checkpoints = newMemorySyncCheckpointStore()
+ }
+ return &SyncRunner{config: config, taskService: taskService, sink: sink, idResolver: idResolver, queue: queue, checkpoints: checkpoints}
}
// Run executes all sync batches and commits the final waterline.
-func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {
+func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) (string, error) {
// sink is nil means this syncer task cannot write it to document, it will fail anyway
if r.sink == nil {
- return errors.New("document sink is not configured")
+ return "", errors.New("document sink is not configured")
}
- windowEnd := time.Now().UTC()
-
var windowStart *time.Time // = nil if it is `Full synchronisation`
if !service.IsFromBeginning(taskContext.Task.FromBeginning) {
windowStart = taskContext.Task.PollRangeStart
}
+ checkpointState, err := r.prepareCheckpoint(ctx, taskContext, windowStart, time.Now().UTC())
+ if err != nil {
+ return "", err
+ }
+ sourceType := service.SourceType(taskContext.Connector.Source, taskContext.Connector.ID)
+ fingerprints, err := r.idResolver.ListFingerprintsBySourceType(ctx, taskContext.Knowledgebase.ID, sourceType)
+ if err != nil {
+ return "", err
+ }
session, err := connector.OpenSync(ctx, syncerconnector.SyncRequest{
TaskID: taskContext.Task.ID,
ConnectorID: taskContext.Connector.ID,
KBID: taskContext.Knowledgebase.ID,
- FromBeginning: windowStart == nil,
- WindowStart: windowStart,
- WindowEnd: windowEnd,
+ SourceType: sourceType,
+ Fingerprints: fingerprints,
+ FromBeginning: checkpointState.WindowStart == nil,
+ WindowStart: checkpointState.WindowStart,
+ WindowEnd: checkpointState.WindowEnd,
+ Resume: checkpointState.Checkpoint,
})
if err != nil {
- return err
+ return "", err
}
defer session.Close()
// prepare sourceType, waterline, stats, resultChan
- sourceType := service.SourceType(taskContext.Connector.Source, taskContext.Connector.ID)
- candidateEnd := windowStart // the waterLine that will write to DB
- stats := service.SyncStats{} // count `add`, `updated`, `skipped`
+ stats := statsFromCheckpointState(checkpointState) // count `add`, `updated`, `skipped`
resultChans := make([]<-chan syncJobResult, 0)
for {
// check if task has been canceled
if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {
- return err
+ return "", err
}
// get a batch of files
@@ -89,24 +104,40 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContex
break
}
if nextErr != nil {
- return nextErr
- }
-
- for _, doc := range batch.Documents {
- if candidateEnd == nil || doc.UpdatedAt.After(*candidateEnd) {
- updatedAt := doc.UpdatedAt
- candidateEnd = &updatedAt // use `max_update` to push `waterLine`
+ if err = r.collectResults(ctx, taskContext.Task.ID, resultChans, &checkpointState, &stats); err != nil {
+ return "", err
}
+ return "", nextErr
}
+
// a batch, a syncJob
resultChan, err := r.submitBatch(ctx, taskContext, sourceType, session, batch)
if err != nil {
- return err
+ return "", err
}
resultChans = append(resultChans, resultChan)
}
- // run sync Job
+ if err = r.collectResults(ctx, taskContext.Task.ID, resultChans, &checkpointState, &stats); err != nil {
+ return "", err
+ }
+
+ if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {
+ return "", err
+ }
+ nextTaskID, err := r.taskService.CompleteSync(ctx, taskContext, checkpointState.WindowEnd, stats)
+ if err != nil {
+ return "", err
+ }
+ if err = r.checkpoints.DeleteSyncCheckpoint(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {
+ common.Warn("delete sync checkpoint failed", zap.Error(err))
+ } else {
+ common.Info("delete sync checkpoint completed", zap.String("task_id", taskContext.Task.ID))
+ }
+ return nextTaskID, nil
+}
+
+func (r *SyncRunner) collectResults(ctx context.Context, taskID string, resultChans []<-chan syncJobResult, checkpointState *syncerconnector.SyncCheckpointState, stats *service.SyncStats) error {
var firstErr error
for _, resultChan := range resultChans {
var jobResult syncJobResult
@@ -119,31 +150,97 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContex
if jobResult.err != nil && firstErr == nil {
firstErr = jobResult.err
}
+ if firstErr == nil && jobResult.checkpoint != nil {
+ checkpointState.Checkpoint = cloneSyncCheckpoint(jobResult.checkpoint)
+ checkpointState.NextCommitSeq++
+ applyStatsToCheckpointState(checkpointState, *stats)
+ if err := r.checkpoints.SaveSyncCheckpoint(ctx, taskID, *checkpointState); err != nil {
+ return err
+ }
+ }
}
if firstErr != nil {
return firstErr
}
+ return nil
+}
- if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {
- return err
+func (r *SyncRunner) prepareCheckpoint(ctx context.Context, taskContext service.SyncTaskContext, windowStart *time.Time, windowEnd time.Time) (syncerconnector.SyncCheckpointState, error) {
+ state, err := r.checkpoints.LoadSyncCheckpoint(ctx, taskContext.Task.ID)
+ if err != nil {
+ return syncerconnector.SyncCheckpointState{}, err
}
- if candidateEnd == nil {
- candidateEnd = &windowEnd
+ if state != nil && !state.WindowEnd.IsZero() && state.TaskID == taskContext.Task.ID && state.ConnectorID == taskContext.Connector.ID && state.KBID == taskContext.Knowledgebase.ID {
+ if state.Version == 0 {
+ state.Version = 1
+ }
+ if state.NextCommitSeq == 0 {
+ state.NextCommitSeq = 1
+ }
+ return *state, nil
+ }
+ // if checkpoint is not exist
+ initial := syncerconnector.SyncCheckpointState{
+ Version: 1,
+ TaskID: taskContext.Task.ID,
+ ConnectorID: taskContext.Connector.ID,
+ KBID: taskContext.Knowledgebase.ID,
+ WindowStart: windowStart,
+ WindowEnd: windowEnd,
+ NextCommitSeq: 1,
}
- return r.taskService.CompleteSync(ctx, taskContext, *candidateEnd, stats)
+ if err = r.checkpoints.SaveSyncCheckpoint(ctx, taskContext.Task.ID, initial); err != nil {
+ return syncerconnector.SyncCheckpointState{}, err
+ }
+ return initial, nil
+}
+
+func statsFromCheckpointState(state syncerconnector.SyncCheckpointState) service.SyncStats {
+ return service.SyncStats{
+ Added: state.Added,
+ Updated: state.Updated,
+ Skipped: state.Skipped,
+ ErrorCount: state.ErrorCount,
+ ErrorMsg: state.ErrorMsg,
+ }
+}
+
+// applyStatsToCheckpointState store checkpoint's state data
+func applyStatsToCheckpointState(state *syncerconnector.SyncCheckpointState, stats service.SyncStats) {
+ state.Added = stats.Added
+ state.Updated = stats.Updated
+ state.Skipped = stats.Skipped
+ state.ErrorCount = stats.ErrorCount
+ state.ErrorMsg = stats.ErrorMsg
}
// submitBatch submits one source batch as one BatchJob.
func (r *SyncRunner) submitBatch(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, batch syncerconnector.SyncBatch) (<-chan syncJobResult, error) {
- resultChan, err := r.queue.Submit(ctx, func(jobCtx context.Context) (service.SyncStats, error) {
+ resultChan, err := r.queue.submit(ctx, func(jobCtx context.Context) (service.SyncStats, error) {
return r.processDocuments(jobCtx, taskContext, sourceType, session, batch.Documents)
- })
+ }, batch.Checkpoint)
if err != nil {
return nil, err
}
return resultChan, nil
}
+// cloneSyncCheckpoint safe clone
+func cloneSyncCheckpoint(checkpoint *syncerconnector.SyncCheckpoint) *syncerconnector.SyncCheckpoint {
+ if checkpoint == nil {
+ return nil
+ }
+ clone := &syncerconnector.SyncCheckpoint{
+ Cursor: checkpoint.Cursor,
+ SourceID: checkpoint.SourceID,
+ }
+ if checkpoint.UpdatedAt != nil {
+ updatedAt := *checkpoint.UpdatedAt
+ clone.UpdatedAt = &updatedAt
+ }
+ return clone
+}
+
// processDocuments
func (r *SyncRunner) processDocuments(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, documents []syncerconnector.SourceDocument) (service.SyncStats, error) {
stats := service.SyncStats{}
diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go
index d41c7735dd..51ae09ce07 100644
--- a/internal/syncer/syncer.go
+++ b/internal/syncer/syncer.go
@@ -49,11 +49,10 @@ type Syncer struct {
}
// NewSyncer creates a server-compatible syncer with default dependencies.
-func NewSyncer(taskWorkerCount int, pollInterval time.Duration) *Syncer {
+func NewSyncer(taskWorkerCount int) *Syncer {
// init the config
config := DefaultConfig()
config.TaskWorkerCount = taskWorkerCount
- config.PollInterval = pollInterval
taskDAO := dao.NewSyncTaskDAO(nil)
registry := syncerconnector.NewRegistry()
@@ -78,15 +77,20 @@ func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, si
})
taskService := service.NewSyncTaskService(taskDAO)
idResolver := service.NewDocumentIDResolver(service.NewGormDocumentStore())
+ checkpoints := SyncCheckpointStore(newMemorySyncCheckpointStore())
+ messageQueue := engine.GetMessageQueueEngine()
+ if store, ok := messageQueue.(SyncCheckpointStore); ok {
+ checkpoints = store
+ }
coordinator := NewTaskCoordinator(TaskCoordinatorConfig{
ItemRetryCount: config.ItemRetryCount,
ItemRetryBaseDelay: config.ItemRetryBaseDelay,
- }, taskService, registry, sink, pruneService, idResolver, executor)
+ }, taskService, registry, sink, pruneService, idResolver, executor, checkpoints)
- scheduler := NewScheduler(config.PollInterval, queue, taskService)
- if broker, ok := engine.GetMessageQueueEngine().(SyncTaskBroker); ok {
- scheduler = NewNATSScheduler(config.PollInterval, queue, taskService, broker)
+ scheduler := NewScheduler(queue, taskService)
+ if broker, ok := messageQueue.(SyncTaskBroker); ok {
+ scheduler = NewNATSScheduler(queue, taskService, broker)
}
return &Syncer{
@@ -94,7 +98,7 @@ func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, si
config: config,
queue: queue,
scheduler: scheduler,
- worker: NewTaskWorker(queue, taskService, coordinator, locker),
+ worker: NewTaskWorker(queue, taskService, coordinator, locker).WithScheduler(scheduler),
executor: executor,
ShutdownCh: make(chan struct{}),
}
@@ -108,11 +112,6 @@ func (s *Syncer) ID() string {
return s.id
}
-// Start launches the scheduler and task workers with a background context.
-func (s *Syncer) Start() error {
- return s.StartContext(context.Background())
-}
-
// StartContext launches the scheduler and task workers.
func (s *Syncer) StartContext(ctx context.Context) error {
if s == nil {
diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go
index a6bec5a3d5..59136280e4 100644
--- a/internal/syncer/syncer_test.go
+++ b/internal/syncer/syncer_test.go
@@ -101,6 +101,14 @@ func (s fakeStore) ListIDs(ctx context.Context, kbID, sourceType string) (map[st
return s.ids, nil
}
+// ListFingerprintsBySourceType returns configured fingerprints.
+func (s fakeStore) ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error) {
+ if s.fingerprints == nil {
+ return map[string]string{}, nil
+ }
+ return s.fingerprints, nil
+}
+
// GetFingerprintsByIDs returns configured fingerprints for requested IDs.
func (s fakeStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {
result := make(map[string]string, len(ids))
@@ -156,26 +164,30 @@ func (h *fakeTaskHandle) counts() (int, int) {
}
type fakeSyncTaskBroker struct {
+ mu sync.Mutex
published []string
- fetched []common.TaskHandle
+ handler func(common.TaskHandle)
}
func (b *fakeSyncTaskBroker) InitSyncerStream() error { return nil }
func (b *fakeSyncTaskBroker) InitSyncerConsumer() error { return nil }
func (b *fakeSyncTaskBroker) PublishSyncerTask(taskID string) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
b.published = append(b.published, taskID)
return nil
}
-func (b *fakeSyncTaskBroker) FetchSyncerTasks(batchSize int) ([]common.TaskHandle, error) {
- if len(b.fetched) == 0 {
- return nil, nil
- }
- if batchSize > len(b.fetched) {
- batchSize = len(b.fetched)
- }
- out := b.fetched[:batchSize]
- b.fetched = b.fetched[batchSize:]
- return out, nil
+func (b *fakeSyncTaskBroker) PublishSyncerTaskWakeup(taskID string) error {
+ return b.PublishSyncerTask(taskID)
+}
+func (b *fakeSyncTaskBroker) SubscribeSyncerTasks(ctx context.Context, handler func(common.TaskHandle)) error {
+ b.handler = handler
+ return nil
+}
+func (b *fakeSyncTaskBroker) publishedIDs() []string {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return append([]string(nil), b.published...)
}
// DeleteDocument records one delete.
@@ -308,8 +320,12 @@ func newTestRegistry(connectors map[string]*connectormock.Connector) *syncerconn
// newCoordinator creates a test coordinator.
func newCoordinator(taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore) *TaskCoordinator {
+ return newCoordinatorWithCheckpoints(taskService, registry, sink, pruneService, store, newMemorySyncCheckpointStore())
+}
+
+func newCoordinatorWithCheckpoints(taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore, checkpoints SyncCheckpointStore) *TaskCoordinator {
executor := NewSyncJobExecutor(SyncJobExecutorConfig{WorkerCount: 16})
- return NewTaskCoordinator(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), executor)
+ return NewTaskCoordinator(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), executor, checkpoints)
}
// TestClaimBlocksSameConnectorKBRunningTasks verifies DB-backed task mutual exclusion.
@@ -354,7 +370,7 @@ func TestClaimAllowsSameConnectorDifferentKB(t *testing.T) {
// TestSchedulerRequiresBroker verifies JetStream is mandatory for the scheduler.
func TestSchedulerRequiresBroker(t *testing.T) {
- scheduler := NewScheduler(time.Hour, make(chan TaskEnvelope, 1), service.NewSyncTaskService(dao.NewSyncTaskDAO(nil)))
+ scheduler := NewScheduler(make(chan TaskEnvelope, 1), service.NewSyncTaskService(dao.NewSyncTaskDAO(nil)))
err := scheduler.Run(t.Context())
if err == nil || !strings.Contains(err.Error(), "NATS broker") {
t.Fatalf("Run error = %v, want missing broker", err)
@@ -367,7 +383,7 @@ func TestNATSSchedulerPublishesDueTasksWithoutClaiming(t *testing.T) {
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
broker := &fakeSyncTaskBroker{}
- scheduler := NewNATSScheduler(time.Hour, make(chan TaskEnvelope, 1), taskService, broker)
+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)
if err := scheduler.publishStartupTasks(t.Context()); err != nil {
t.Fatalf("publish startup tasks: %v", err)
}
@@ -383,8 +399,8 @@ func TestNATSSchedulerPublishesDueTasksWithoutClaiming(t *testing.T) {
}
}
-// TestNATSSchedulerStartupPublishesScheduledTaskWithRefreshFreq verifies startup does not wait for the next refresh window.
-func TestNATSSchedulerStartupPublishesScheduledTaskWithRefreshFreq(t *testing.T) {
+// TestNATSSchedulerStartupDelaysFreshScheduledTask verifies startup keeps refresh windows without periodic DB scans.
+func TestNATSSchedulerStartupDelaysFreshScheduledTask(t *testing.T) {
db := setupSyncerDB(t)
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
now := time.Now()
@@ -401,12 +417,18 @@ func TestNATSSchedulerStartupPublishesScheduledTaskWithRefreshFreq(t *testing.T)
taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
broker := &fakeSyncTaskBroker{}
- scheduler := NewNATSScheduler(time.Hour, make(chan TaskEnvelope, 1), taskService, broker)
+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)
if err := scheduler.publishStartupTasks(t.Context()); err != nil {
t.Fatalf("publish startup tasks: %v", err)
}
- if len(broker.published) != 1 || broker.published[0] != "task-1" {
- t.Fatalf("published = %v", broker.published)
+ if got := broker.publishedIDs(); len(got) != 0 {
+ t.Fatalf("published = %v, want delayed task", got)
+ }
+ scheduler.timerMu.Lock()
+ _, scheduled := scheduler.timers["task-1"]
+ scheduler.timerMu.Unlock()
+ if !scheduled {
+ t.Fatalf("task timer was not scheduled")
}
}
@@ -422,13 +444,13 @@ func TestNATSSchedulerRecoversRunningTasksOnStartup(t *testing.T) {
}
taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
broker := &fakeSyncTaskBroker{}
- scheduler := NewNATSScheduler(time.Hour, make(chan TaskEnvelope, 1), taskService, broker)
+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)
if err := scheduler.publishStartupTasks(t.Context()); err != nil {
t.Fatalf("publish startup tasks: %v", err)
}
- if len(broker.published) != 1 || broker.published[0] != "task-1" {
- t.Fatalf("published = %v", broker.published)
+ if got := broker.publishedIDs(); len(got) != 1 || got[0] != "task-1" {
+ t.Fatalf("published = %v", got)
}
var task entity.SyncLogs
if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
@@ -446,8 +468,8 @@ func TestNATSSchedulerRecoversRunningTasksOnStartup(t *testing.T) {
}
}
-// TestNATSSchedulerPublishesOnlyDueTasks verifies periodic NATS publishing respects refresh windows.
-func TestNATSSchedulerPublishesOnlyDueTasks(t *testing.T) {
+// TestNATSSchedulerStartupPublishesDueTasks verifies startup scan immediately publishes due work.
+func TestNATSSchedulerStartupPublishesDueTasks(t *testing.T) {
db := setupSyncerDB(t)
insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
now := time.Now()
@@ -463,13 +485,13 @@ func TestNATSSchedulerPublishesOnlyDueTasks(t *testing.T) {
}
taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
broker := &fakeSyncTaskBroker{}
- scheduler := NewNATSScheduler(10*time.Millisecond, make(chan TaskEnvelope, 1), taskService, broker)
+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)
- if err := scheduler.publishDueTasks(t.Context()); err != nil {
- t.Fatalf("publish fresh due tasks: %v", err)
+ if err := scheduler.publishStartupTasks(t.Context()); err != nil {
+ t.Fatalf("publish fresh startup tasks: %v", err)
}
- if len(broker.published) != 0 {
- t.Fatalf("fresh task published = %v, want none", broker.published)
+ if got := broker.publishedIDs(); len(got) != 0 {
+ t.Fatalf("fresh task published = %v, want none", got)
}
dueAt := now.Add(-10 * time.Minute)
@@ -480,26 +502,28 @@ func TestNATSSchedulerPublishesOnlyDueTasks(t *testing.T) {
t.Fatalf("set due task update time: %v", err)
}
- if err := scheduler.publishDueTasks(t.Context()); err != nil {
- t.Fatalf("publish due tasks: %v", err)
+ if err := scheduler.publishStartupTasks(t.Context()); err != nil {
+ t.Fatalf("publish due startup tasks: %v", err)
}
- if len(broker.published) != 1 || broker.published[0] != "task-1" {
- t.Fatalf("published = %v, want due task", broker.published)
+ if got := broker.publishedIDs(); len(got) != 1 || got[0] != "task-1" {
+ t.Fatalf("published = %v, want due task", got)
}
}
-// TestNATSSchedulerBuffersFetchedTasks verifies NATS mode stores excess tasks in the local queue.
-func TestNATSSchedulerBuffersFetchedTasks(t *testing.T) {
+// TestNATSSchedulerBuffersPushedTasks verifies enqueueHandle buffers pushed NATS handles in the local queue.
+func TestNATSSchedulerBuffersPushedTasks(t *testing.T) {
taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(nil))
queue := make(chan TaskEnvelope, 2)
- scheduler := NewNATSScheduler(time.Hour, queue, taskService, &fakeSyncTaskBroker{})
+ scheduler := NewNATSScheduler(queue, taskService, &fakeSyncTaskBroker{})
handles := []common.TaskHandle{
&fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-1", TaskType: common.TaskTypeSyncer}},
&fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-2", TaskType: common.TaskTypeSyncer}},
}
- if err := scheduler.enqueueHandles(t.Context(), handles); err != nil {
- t.Fatalf("enqueue handles: %v", err)
+ for _, handle := range handles {
+ if err := scheduler.enqueueHandle(t.Context(), handle); err != nil {
+ t.Fatalf("enqueue handle: %v", err)
+ }
}
if got := scheduler.queueAvailable(); got != 0 {
t.Fatalf("queue capacity while buffered = %d, want 0", got)
@@ -717,7 +741,7 @@ func TestSyncRunnerSubmitsBatchesBeforeWaiting(t *testing.T) {
sink := &fakeSink{delay: 80 * time.Millisecond}
coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{})
taskContext, _ := taskService.GetContext(t.Context(), "task-1")
- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
t.Fatalf("execute: %v", err)
}
if secondBatchAt.Sub(start) >= 70*time.Millisecond {
@@ -740,7 +764,7 @@ func TestSyncRunnerProcessesBatchJobsInParallel(t *testing.T) {
sink := &fakeSink{delay: 80 * time.Millisecond}
coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{})
taskContext, _ := taskService.GetContext(t.Context(), "task-1")
- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
t.Fatalf("execute: %v", err)
}
sink.mu.Lock()
@@ -778,7 +802,7 @@ func TestFingerprintSkipsUnchangedDocument(t *testing.T) {
sink := &fakeSink{}
coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, store)
taskContext, _ := taskService.GetContext(t.Context(), "task-1")
- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
t.Fatalf("execute: %v", err)
}
if sink.callCount() != 0 {
@@ -799,7 +823,7 @@ func TestAutoParseFlagFlowsToSink(t *testing.T) {
sink := &fakeSink{}
coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{})
taskContext, _ := taskService.GetContext(t.Context(), "task-1")
- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
t.Fatalf("execute: %v", err)
}
if sink.autoParseByDoc["source-1"] {
@@ -818,7 +842,7 @@ func TestCompleteSyncSchedulesNextRun(t *testing.T) {
connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("x"), UpdatedAt: now}}}}}
coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{})
taskContext, _ := taskService.GetContext(t.Context(), "task-1")
- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {
t.Fatalf("execute: %v", err)
}
@@ -909,12 +933,13 @@ func TestSyncRunnerResultWaitHonorsCancel(t *testing.T) {
}
connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}
queue := &SyncJobQueue{taskID: "task-1", jobs: make(chan *syncJob, 1)}
- runner := NewSyncRunner(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, &fakeSink{}, service.NewDocumentIDResolver(fakeStore{}), queue)
+ runner := NewSyncRunner(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, &fakeSink{}, service.NewDocumentIDResolver(fakeStore{}), queue, newMemorySyncCheckpointStore())
ctx, cancel := context.WithCancel(context.Background())
result := make(chan error, 1)
go func() {
- result <- runner.Run(ctx, taskContext, connector)
+ _, err := runner.Run(ctx, taskContext, connector)
+ result <- err
}()
select {
@@ -957,6 +982,272 @@ func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {
}
}
+// TestTransientFailureReschedulesTask verifies retryable source failures keep the same task scheduled.
+func TestTransientFailureReschedulesTask(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ connector := &connectormock.Connector{SyncErrAt: 1}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusSchedule {
+ t.Fatalf("status = %s, want schedule", task.Status)
+ }
+ if task.ErrorCount != 1 {
+ t.Fatalf("error_count = %d, want 1", task.ErrorCount)
+ }
+ if !strings.Contains(task.ErrorMsg, "unexpected EOF") {
+ t.Fatalf("error_msg = %q, want unexpected EOF", task.ErrorMsg)
+ }
+}
+
+// TestTransientFailureFailsAfterThreeRetries verifies retryable errors become terminal after the retry budget.
+func TestTransientFailureFailsAfterThreeRetries(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(2)}).Error; err != nil {
+ t.Fatalf("mark running: %v", err)
+ }
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ connector := &connectormock.Connector{SyncErrAt: 1}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusFail {
+ t.Fatalf("status = %s, want fail", task.Status)
+ }
+ if task.ErrorCount != 3 {
+ t.Fatalf("error_count = %d, want 3", task.ErrorCount)
+ }
+ if !strings.Contains(task.ErrorMsg, "failed after 3 transient retries") || !strings.Contains(task.ErrorMsg, "unexpected EOF") {
+ t.Fatalf("error_msg = %q", task.ErrorMsg)
+ }
+}
+
+// TestTransientFetchFailureSavesCompletedBatchCheckpoint verifies source failures do not discard completed batch progress.
+func TestTransientFetchFailureSavesCompletedBatchCheckpoint(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ checkpoints := newMemorySyncCheckpointStore()
+ firstTime := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+ connector := &connectormock.Connector{
+ SyncErrAt: 2,
+ SyncBatches: []syncerconnector.SyncBatch{{
+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("a"), UpdatedAt: firstTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &firstTime, SourceID: "source-1"},
+ }},
+ }
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}, checkpoints), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("load checkpoint: %v", err)
+ }
+ if state == nil || state.Checkpoint == nil {
+ t.Fatalf("checkpoint was not saved")
+ }
+ if state.Checkpoint.Cursor != "cursor-1" || state.Checkpoint.SourceID != "source-1" {
+ t.Fatalf("checkpoint = %+v, want cursor-1/source-1", state.Checkpoint)
+ }
+ if state.Added != 1 {
+ t.Fatalf("checkpoint added = %d, want 1", state.Added)
+ }
+
+ var task entity.SyncLogs
+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusSchedule {
+ t.Fatalf("status = %s, want schedule", task.Status)
+ }
+}
+
+// TestSyncCheckpointDeletedAfterSuccess verifies completed tasks remove their checkpoint.
+func TestSyncCheckpointDeletedAfterSuccess(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ checkpoints := newMemorySyncCheckpointStore()
+
+ firstTime := time.Date(2026, 8, 11, 1, 0, 0, 0, time.UTC)
+ secondTime := time.Date(2026, 8, 11, 1, 1, 0, 0, time.UTC)
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{
+ {
+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("a"), UpdatedAt: firstTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &firstTime, SourceID: "source-1"},
+ },
+ {
+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-2", Blob: []byte("b"), UpdatedAt: secondTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-2", UpdatedAt: &secondTime, SourceID: "source-2"},
+ },
+ }}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}, checkpoints), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("load checkpoint: %v", err)
+ }
+ if state != nil {
+ t.Fatalf("checkpoint = %+v, want deleted", state)
+ }
+}
+
+// TestSyncCheckpointStopsBeforeFailedBatch verifies failed batches do not advance the resume point.
+func TestSyncCheckpointStopsBeforeFailedBatch(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ checkpoints := newMemorySyncCheckpointStore()
+
+ firstTime := time.Date(2026, 8, 11, 1, 0, 0, 0, time.UTC)
+ secondTime := time.Date(2026, 8, 11, 1, 1, 0, 0, time.UTC)
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{
+ {
+ Documents: []syncerconnector.SourceDocument{{SourceID: "good", Blob: []byte("a"), UpdatedAt: firstTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &firstTime, SourceID: "good"},
+ },
+ {
+ Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("b"), UpdatedAt: secondTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-2", UpdatedAt: &secondTime, SourceID: "bad"},
+ },
+ }}
+ sink := &fakeSink{errBySourceID: map[string]error{"bad": errors.New("boom")}}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, checkpoints), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("load checkpoint: %v", err)
+ }
+ if state == nil || state.Checkpoint == nil {
+ t.Fatalf("checkpoint was not saved")
+ }
+ if state.Checkpoint.Cursor != "cursor-1" {
+ t.Fatalf("checkpoint cursor = %q, want cursor-1", state.Checkpoint.Cursor)
+ }
+ if state.NextCommitSeq != 2 {
+ t.Fatalf("next commit seq = %d, want 2", state.NextCommitSeq)
+ }
+ if state.Added != 1 || state.Updated != 0 {
+ t.Fatalf("checkpoint stats added/updated = %d/%d, want 1/0", state.Added, state.Updated)
+ }
+
+ if err = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error; err != nil {
+ t.Fatalf("mark running: %v", err)
+ }
+ thirdTime := secondTime.Add(time.Minute)
+ resumeConnector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{
+ Documents: []syncerconnector.SourceDocument{{SourceID: "second-good", Blob: []byte("c"), UpdatedAt: thirdTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-3", UpdatedAt: &thirdTime, SourceID: "second-good"},
+ }}}
+ resumeWorker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": resumeConnector}), &fakeSink{}, nil, fakeStore{}, checkpoints), NewConnectorLock())
+ resumeWorker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+ if len(resumeConnector.SyncRequests) != 1 {
+ t.Fatalf("sync requests = %d, want 1", len(resumeConnector.SyncRequests))
+ }
+ request := resumeConnector.SyncRequests[0]
+ if request.Resume == nil || request.Resume.Cursor != "cursor-1" || request.Resume.SourceID != "good" {
+ t.Fatalf("resume = %+v, want cursor-1/good", request.Resume)
+ }
+ if !request.WindowEnd.Equal(state.WindowEnd) {
+ t.Fatalf("window end = %s, want %s", request.WindowEnd, state.WindowEnd)
+ }
+ var task entity.SyncLogs
+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load completed task: %v", err)
+ }
+ if task.Status != dao.SyncStatusDone {
+ t.Fatalf("status = %s, want done", task.Status)
+ }
+ if task.TotalDocsIndexed != 2 {
+ t.Fatalf("total docs indexed = %d, want 2", task.TotalDocsIndexed)
+ }
+}
+
+// TestSyncRunnerClampsWaterlineToWindowEnd verifies future source timestamps do not poison the next window.
+func TestSyncRunnerClampsWaterlineToWindowEnd(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ start := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+ if err := db.Model(&entity.SyncLogs{}).
+ Where("id = ?", "task-1").
+ Updates(map[string]any{"status": dao.SyncStatusRunning, "poll_range_start": &start}).Error; err != nil {
+ t.Fatalf("mark running: %v", err)
+ }
+
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ future := time.Date(2036, 6, 28, 9, 44, 0, 0, time.UTC)
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{
+ Documents: []syncerconnector.SourceDocument{{SourceID: "future", Blob: []byte("future"), UpdatedAt: future}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-future", UpdatedAt: &future, SourceID: "future"},
+ }}}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())
+ before := time.Now().UTC()
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+ after := time.Now().UTC().Add(time.Second)
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.PollRangeEnd == nil {
+ t.Fatalf("poll_range_end is nil")
+ }
+ if task.PollRangeEnd.Equal(future) || task.PollRangeEnd.After(after) || task.PollRangeEnd.Before(before.Add(-time.Second)) {
+ t.Fatalf("poll_range_end = %s, want clamped to run window around %s - %s", task.PollRangeEnd, before, after)
+ }
+}
+
+// TestFullSyncWaterlineUsesWindowEnd verifies successful full sync does not keep the waterline at the latest source timestamp.
+func TestFullSyncWaterlineUsesWindowEnd(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ fromBeginning := "1"
+ if err := db.Model(&entity.SyncLogs{}).
+ Where("id = ?", "task-1").
+ Updates(map[string]any{"status": dao.SyncStatusRunning, "from_beginning": &fromBeginning}).Error; err != nil {
+ t.Fatalf("mark running: %v", err)
+ }
+
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ oldSourceTime := time.Date(2025, 5, 17, 12, 39, 23, 0, time.UTC)
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{
+ Documents: []syncerconnector.SourceDocument{{SourceID: "old", Blob: []byte("old"), UpdatedAt: oldSourceTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-old", UpdatedAt: &oldSourceTime, SourceID: "old"},
+ }}}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())
+ before := time.Now().UTC()
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+ after := time.Now().UTC().Add(time.Second)
+
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.PollRangeEnd == nil {
+ t.Fatalf("poll_range_end is nil")
+ }
+ if task.PollRangeEnd.Equal(oldSourceTime) || task.PollRangeEnd.Before(before.Add(-time.Second)) || task.PollRangeEnd.After(after) {
+ t.Fatalf("poll_range_end = %s, want run window around %s - %s", task.PollRangeEnd, before, after)
+ }
+}
+
// TestRecoverStaleRunningTasks verifies timeout recovery.
func TestRecoverStaleRunningTasks(t *testing.T) {
db := setupSyncerDB(t)
diff --git a/internal/syncer/task_coordinator.go b/internal/syncer/task_coordinator.go
index 8b7ffd1419..5a3b61abe6 100644
--- a/internal/syncer/task_coordinator.go
+++ b/internal/syncer/task_coordinator.go
@@ -47,10 +47,16 @@ type TaskCoordinator struct {
pruneService *service.SyncPruneService
idResolver *service.DocumentIDResolver
executor *SyncJobExecutor
+ checkpoints SyncCheckpointStore
+}
+
+// TaskOutcome describes post-run scheduling work for a completed task.
+type TaskOutcome struct {
+ NextTaskID string
}
// NewTaskCoordinator creates a coordinator for one claimed task at a time.
-func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTaskService, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService, idResolver *service.DocumentIDResolver, executor *SyncJobExecutor) *TaskCoordinator {
+func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTaskService, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService, idResolver *service.DocumentIDResolver, executor *SyncJobExecutor, checkpoints SyncCheckpointStore) *TaskCoordinator {
if config.ItemRetryCount <= 0 {
config.ItemRetryCount = 1
}
@@ -61,39 +67,44 @@ func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncT
if executor == nil {
panic("task coordinator executor must not be nil")
}
+ if checkpoints == nil {
+ checkpoints = newMemorySyncCheckpointStore()
+ }
- return &TaskCoordinator{config: config, taskService: taskService, registry: registry, sink: sink, pruneService: pruneService, idResolver: idResolver, executor: executor}
+ return &TaskCoordinator{config: config, taskService: taskService, registry: registry, sink: sink, pruneService: pruneService, idResolver: idResolver, executor: executor, checkpoints: checkpoints}
}
// Execute dispatches a sync_logs task by task type.
-func (c *TaskCoordinator) Execute(ctx context.Context, taskContext service.SyncTaskContext, lease ConnectorLockLease) error {
+func (c *TaskCoordinator) Execute(ctx context.Context, taskContext service.SyncTaskContext, lease ConnectorLockLease) (TaskOutcome, error) {
runCtx, cancel := context.WithDeadline(ctx, taskExecutionDeadline(time.Now(), taskContext, lease))
defer cancel()
ctx = runCtx
connector, err := c.registry.Open(ctx, taskContext)
if err != nil {
- return err
+ return TaskOutcome{}, err
}
if err = connector.Validate(ctx); err != nil {
- return err
+ return TaskOutcome{}, err
}
switch taskContext.Task.TaskType {
case service.TaskTypeSync:
queue, err := c.executor.RegisterTask(ctx, taskContext.Task.ID)
if err != nil {
- return err
+ return TaskOutcome{}, err
}
defer queue.Close()
- runner := NewSyncRunner(c.config, c.taskService, c.sink, c.idResolver, queue)
- return runner.Run(ctx, taskContext, connector)
+ runner := NewSyncRunner(c.config, c.taskService, c.sink, c.idResolver, queue, c.checkpoints)
+ nextTaskID, err := runner.Run(ctx, taskContext, connector)
+ return TaskOutcome{NextTaskID: nextTaskID}, err
case service.TaskTypePrune:
runner := NewPruneRunner(c.taskService, c.pruneService)
- return runner.Run(ctx, taskContext, connector)
+ nextTaskID, err := runner.Run(ctx, taskContext, connector)
+ return TaskOutcome{NextTaskID: nextTaskID}, err
default:
- return fmt.Errorf("unsupported sync task type %q", taskContext.Task.TaskType)
+ return TaskOutcome{}, fmt.Errorf("unsupported sync task type %q", taskContext.Task.TaskType)
}
}
diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.go
index 044a5c17bc..caffa9ac3c 100644
--- a/internal/syncer/task_worker.go
+++ b/internal/syncer/task_worker.go
@@ -24,6 +24,8 @@ import (
"ragflow/internal/service"
"sync"
"time"
+
+ "go.uber.org/zap"
)
// TaskWorker consumes claimed task envelopes and runs coordinators.
@@ -32,6 +34,7 @@ type TaskWorker struct {
taskService *service.SyncTaskService
coordinator *TaskCoordinator
locker ConnectorLocker
+ scheduler *Scheduler
}
// NewTaskWorker creates a bounded task worker pool.
@@ -39,6 +42,12 @@ func NewTaskWorker(queue <-chan TaskEnvelope, taskService *service.SyncTaskServi
return &TaskWorker{queue: queue, taskService: taskService, coordinator: coordinator, locker: locker}
}
+// WithScheduler attaches the event scheduler used for one-shot task publishing.
+func (w *TaskWorker) WithScheduler(scheduler *Scheduler) *TaskWorker {
+ w.scheduler = scheduler
+ return w
+}
+
// Run starts worker goroutines and blocks until context cancellation.
func (w *TaskWorker) Run(ctx context.Context, concurrency int) {
var wg sync.WaitGroup
@@ -107,21 +116,38 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
lease, locked := w.locker.TryLock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)
if !locked {
_ = w.taskService.RescheduleClaimed(ctx, taskContext.Task.ID)
+ w.scheduleRetry(ctx, taskContext.Task.ID, 3*time.Second)
ackEnvelope(envelope)
return
}
defer w.locker.Unlock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)
startedAt := time.Now()
- if err = w.coordinator.Execute(ctx, taskContext, lease); err != nil { // execute the task(sync/ prune)
- logSyncTaskDuration(taskContext, startedAt)
- if errors.Is(err, errSyncTaskCanceled) { // task is canceled by the user
+ outcome, err := w.coordinator.Execute(ctx, taskContext, lease)
+ if err != nil { // execute the task(sync/ prune)
+ logSyncTaskDuration(taskContext, startedAt) // test sync task run time
+ if errors.Is(err, errSyncTaskCanceled) { // task is canceled by the user
ackEnvelope(envelope)
return
}
if ctx.Err() != nil { // the task is canceled by system, this need to rerun
_ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID)
- nackEnvelope(envelope)
+ w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)
+ ackEnvelope(envelope)
+ return
+ }
+ if isTransientSyncError(err) {
+ attempts, failed, transientErr := w.taskService.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, err, maxTransientTaskRetries)
+ if transientErr != nil {
+ _ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID)
+ nackEnvelope(envelope)
+ return
+ }
+ logTransientSyncRetry(taskContext, attempts, failed, err)
+ if !failed {
+ w.scheduleRetry(ctx, taskContext.Task.ID, transientRetryDelay(attempts))
+ }
+ ackEnvelope(envelope)
return
}
if failErr := w.taskService.Fail(ctx, taskContext.Task.ID, taskContext.Connector.ID, fmt.Errorf("sync task failed: %w", err)); failErr != nil {
@@ -133,9 +159,64 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
return
}
logSyncTaskDuration(taskContext, startedAt) // Todo delete soon
+ w.scheduleNext(ctx, outcome.NextTaskID)
ackEnvelope(envelope)
}
+func (w *TaskWorker) scheduleNext(ctx context.Context, taskID string) {
+ if w.scheduler == nil || taskID == "" {
+ return
+ }
+ task, err := w.taskService.GetScheduledTask(ctx, taskID)
+ if err != nil {
+ common.Warn("syncer schedule next task lookup failed", zap.String("task_id", taskID), zap.Error(err))
+ return
+ }
+ if err = w.scheduler.ScheduleTask(ctx, task); err != nil {
+ common.Warn("syncer schedule next task failed", zap.String("task_id", taskID), zap.Error(err))
+ }
+}
+
+// scheduleRetry schedule retry when web error occurred
+func (w *TaskWorker) scheduleRetry(ctx context.Context, taskID string, delay time.Duration) {
+ if w.scheduler == nil || taskID == "" {
+ return
+ }
+ if err := w.scheduler.ScheduleTaskAfter(ctx, taskID, delay); err != nil {
+ common.Warn("syncer retry task publish failed", zap.String("task_id", taskID), zap.Duration("delay", delay), zap.Error(err))
+ }
+}
+
+// transientRetryDelay return retry delay
+func transientRetryDelay(attempts int64) time.Duration {
+ if attempts < 1 {
+ attempts = 1
+ }
+
+ shift := attempts - 1
+ if shift > 5 {
+ shift = 5
+ }
+ return time.Duration(1<<shift) * 30 * time.Second
+}
+
+func logTransientSyncRetry(taskContext service.SyncTaskContext, attempts int64, failed bool, err error) {
+ message := "sync task transient retry scheduled"
+ if failed {
+ message = "sync task failed after transient retries"
+ }
+ common.Warn(
+ message,
+ zap.String("task_id", taskContext.Task.ID),
+ zap.String("connector_id", taskContext.Connector.ID),
+ zap.String("kb_id", taskContext.Knowledgebase.ID),
+ zap.String("source", taskContext.Connector.Source),
+ zap.Int64("attempts", attempts),
+ zap.Int64("max_retries", maxTransientTaskRetries),
+ zap.Error(err),
+ )
+}
+
func ackEnvelope(envelope TaskEnvelope) {
if envelope.Handle != nil {
_ = envelope.Handle.Ack()
diff --git a/internal/syncer/transient_error.go b/internal/syncer/transient_error.go
new file mode 100644
index 0000000000..3c6047e071
--- /dev/null
+++ b/internal/syncer/transient_error.go
@@ -0,0 +1,56 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "errors"
+ "io"
+ "net"
+ "ragflow/internal/service"
+ "strings"
+)
+
+// maxTransientTaskRetries max retry time when network error occur
+const maxTransientTaskRetries int64 = 3
+
+// isTransientSyncError reports whether a task-level sync error is worth retrying.
+func isTransientSyncError(err error) bool {
+ if err == nil {
+ return false
+ }
+ if service.IsRetryable(err) || errors.Is(err, io.ErrUnexpectedEOF) {
+ return true
+ }
+
+ var netErr net.Error
+ if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) {
+ return true
+ }
+
+ message := strings.ToLower(err.Error())
+ return strings.Contains(message, "unexpected eof") ||
+ strings.Contains(message, "connection reset") ||
+ strings.Contains(message, "connection refused") ||
+ strings.Contains(message, "timeout") ||
+ strings.Contains(message, "temporary") ||
+ strings.Contains(message, "too many requests") ||
+ strings.Contains(message, "http 429") ||
+ strings.Contains(message, "http 500") ||
+ strings.Contains(message, "http 502") ||
+ strings.Contains(message, "http 503") ||
+ strings.Contains(message, "http 504")
+}