RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat[Go]: resuming transmission from the point of interruption during data source synchronisation (#18176)
Haruko38696f7c39753a9
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex e7c6fab1db..900913e5e3 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -633,15 +633,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArgfunc runSyncer(ctx context.Context, cancel context.CancelFunc, args *serverArgs) error {globalConfig := server.GetConfig()syncerConfig := globalConfig.GetSyncerConfig()- fileSyncer := syncer.NewSyncer(syncerConfig.MaxConcurrentSyncs, time.Duration(syncerConfig.SyncInterval)*time.Second)+ fileSyncer := syncer.NewSyncer(syncerConfig.MaxConcurrentSyncs)- go func() {- err := fileSyncer.Start()- if err != nil {- common.Error("Failed to initialize file syncer", err)- return- }- }()+ if err := fileSyncer.StartContext(ctx); err != nil {+ common.Error("Failed to initialize file syncer", err)+ return err+ }common.Info("\n _______ __ _____\n" +" / ____(_) /__ / ___/__ ______ ________ _____\n" +diff --git a/conf/service_conf.yaml b/conf/service_conf.yamlindex db12d36bcb..fa181efb82 100644--- a/conf/service_conf.yaml+++ b/conf/service_conf.yaml@@ -72,7 +72,6 @@ ingestor:compiler_pool_size: 0file_syncer:max_concurrent_syncs: 5- sync_interval: 3user_default_llm:default_models:embedding_model:diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.templateindex 745b0def57..b60e1317c4 100644--- a/docker/service_conf.yaml.template+++ b/docker/service_conf.yaml.template@@ -90,7 +90,6 @@ ingestor:compiler_pool_size: 0file_syncer:max_concurrent_syncs: 5- sync_interval: 3user_default_llm:default_models:embedding_model:diff --git a/internal/dao/connector.go b/internal/dao/connector.goindex e7dfd5b5fe..fff493bf7e 100644--- a/internal/dao/connector.go+++ b/internal/dao/connector.go@@ -25,6 +25,7 @@ import ("time""gorm.io/gorm"+ "gorm.io/gorm/clause")// ConnectorDAO connector data access object@@ -105,21 +106,26 @@ type DatasetConnectorLink struct {}// LinkDatasetConnectors syncs connector2kb rows for a dataset.-func (dao *ConnectorDAO) LinkDatasetConnectors(ctx context.Context, db *gorm.DB, kbID string, connectors []DatasetConnectorLink) error {- return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+func (dao *ConnectorDAO) LinkDatasetConnectors(ctx context.Context, db *gorm.DB, kbID string, connectors []DatasetConnectorLink) ([]string, error) {+ var taskIDs []string+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {var kb entity.Knowledgebase- if err := db.WithContext(ctx).Select("tenant_id").Where("id = ? AND status = ?", kbID, string(entity.StatusValid)).First(&kb).Error; err != nil {+ if err := tx.WithContext(ctx).Select("tenant_id").Where("id = ? AND status = ?", kbID, string(entity.StatusValid)).First(&kb).Error; err != nil {return err}- return dao.LinkDatasetConnectorsTx(ctx, tx, kbID, kb.TenantID, connectors)+ var err error+ taskIDs, err = dao.LinkDatasetConnectorsTx(ctx, tx, kbID, kb.TenantID, connectors)+ return err})+ return taskIDs, err}// LinkDatasetConnectorsTx syncs connector2kb rows using the caller's transaction.-func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.DB, kbID, tenantID string, connectors []DatasetConnectorLink) error {+func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.DB, kbID, tenantID string, connectors []DatasetConnectorLink) ([]string, error) {+ var taskIDs []stringvar existing []entity.Connector2Kbif err := tx.WithContext(ctx).Where("kb_id = ?", kbID).Find(&existing).Error; err != nil {- return err+ return nil, err}oldConnectorIDs := make(map[string]entity.Connector2Kb, len(existing))@@ -138,16 +144,16 @@ func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.Dvar fullConnector entity.Connectorif err := tx.WithContext(ctx).Where("id = ? AND tenant_id = ?", connector.ID, tenantID).First(&fullConnector).Error; err != nil {if IsNotFoundErr(err) {- return fmt.Errorf("%w: %s", errConnectorNotAccessible, connector.ID)+ return nil, fmt.Errorf("%w: %s", errConnectorNotAccessible, connector.ID)}- return err+ return nil, err}if _, ok := oldConnectorIDs[connector.ID]; ok {if err := tx.WithContext(ctx).Model(&entity.Connector2Kb{}).Where("connector_id = ? AND kb_id = ?", connector.ID, kbID).Update("auto_parse", autoParse).Error; err != nil {- return err+ return nil, err}continue}@@ -158,17 +164,21 @@ func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.DKbID: kbID,AutoParse: autoParse,}).Error; err != nil {- return err+ return nil, err}- if _, err := scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypeSync, true); err != nil {- return err+ taskID, err := scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypeSync, true)+ if err != nil {+ return nil, err}+ taskIDs = append(taskIDs, taskID)if connectorConfigBool(fullConnector.Config, "sync_deleted_files") {- if _, err := scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypePrune, false); err != nil {- return err+ taskID, err = scheduleConnectorTask(ctx, tx, connector.ID, kbID, connectorTaskTypePrune, false)+ if err != nil {+ return nil, err}+ taskIDs = append(taskIDs, taskID)}}@@ -178,16 +188,16 @@ func (dao *ConnectorDAO) LinkDatasetConnectorsTx(ctx context.Context, tx *gorm.D}if err := tx.WithContext(ctx).Where("kb_id = ? AND connector_id = ?", kbID, connectorID).Delete(&entity.Connector2Kb{}).Error; err != nil {- return err+ return nil, err}if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).Where("connector_id = ? AND kb_id = ? AND status IN ?", connectorID, kbID, []string{string(entity.TaskStatusSchedule), string(entity.TaskStatusRunning)}).Update("status", string(entity.TaskStatusCancel)).Error; err != nil {- return err+ return nil, err}}- return nil+ return taskIDs, nil}// GetByID get connector by ID@@ -270,9 +280,15 @@ func (dao *ConnectorDAO) ListDocumentsByKBAndSourceType(ctx context.Context, db}// RebuildConnector replaces old connector documents with scheduled sync tasks.-func (dao *ConnectorDAO) RebuildConnector(ctx context.Context, db *gorm.DB, connector *entity.Connector, kbID string, documents []*entity.Document) ([]string, error) {- taskIDs := []string{}+func (dao *ConnectorDAO) RebuildConnector(ctx context.Context, db *gorm.DB, connector *entity.Connector, kbID string, documents []*entity.Document) ([]string, []string, error) {+ var taskIDs []string+ var oldSyncTaskIDs []stringerr := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).+ Where("connector_id = ? AND kb_id = ? AND task_type = ?", connector.ID, kbID, connectorTaskTypeSync).+ Pluck("id", &oldSyncTaskIDs).Error; err != nil {+ return err+ }if err := tx.WithContext(ctx).Where("connector_id = ? AND kb_id = ?", connector.ID, kbID).Delete(&entity.SyncLogs{}).Error; err != nil {return err}@@ -351,7 +367,38 @@ func (dao *ConnectorDAO) RebuildConnector(ctx context.Context, db *gorm.DB, conn}return nil})- return taskIDs, err+ return taskIDs, oldSyncTaskIDs, err+}++// ResumeFailedSyncTask schedules one failed sync task for checkpoint resume.+func (dao *ConnectorDAO) ResumeFailedSyncTask(ctx context.Context, db *gorm.DB, connectorID, kbID, taskID string) error {+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var task entity.SyncLogs+ query := tx.WithContext(ctx)+ if tx.Dialector.Name() != "sqlite" {+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})+ }+ if err := query.+ Where("id = ? AND connector_id = ? AND kb_id = ? AND task_type = ? AND status = ?", taskID, connectorID, kbID, connectorTaskTypeSync, string(entity.TaskStatusFail)).+ First(&task).Error; err != nil {+ return err+ }++ now := time.Now().Local()+ if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).+ Where("id = ? AND status = ?", taskID, string(entity.TaskStatusFail)).+ Updates(map[string]any{+ "status": string(entity.TaskStatusSchedule),+ "error_msg": "",+ "error_count": int64(0),+ "time_started": &now,+ }).Error; err != nil {+ return err+ }+ return tx.WithContext(ctx).Model(&entity.Connector{}).+ Where("id = ?", connectorID).+ Update("status", string(entity.TaskStatusSchedule)).Error+ })}const (diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.goindex 00825859b1..2d2bc953bc 100644--- a/internal/dao/sync_task.go+++ b/internal/dao/sync_task.go@@ -19,6 +19,7 @@ package daoimport ("context""errors"+ "fmt""ragflow/internal/entity""ragflow/internal/utility""time"@@ -92,8 +93,16 @@ type dueSyncTaskRow struct {ConnectorConfig entity.JSONMap `gorm:"column:connector_config"`}-// ListDueTasks returns due schedule tasks across SYNC and PRUNE.-func (d *SyncTaskDAO) ListDueTasks(ctx context.Context, now time.Time, limit int) ([]entity.SyncLogs, error) {+// ScheduledSyncTask contains one scheduled task and its connector scheduling settings.+type ScheduledSyncTask struct {+ entity.SyncLogs+ ConnectorRefreshFreq int64+ ConnectorPruneFreq int64+ ConnectorConfig entity.JSONMap+}++// ListScheduledTasks returns scheduled tasks with connector scheduling settings.+func (d *SyncTaskDAO) ListScheduledTasks(ctx context.Context, limit int) ([]ScheduledSyncTask, error) {var rows []dueSyncTaskRowif err := d.db.WithContext(ctx).Model(&entity.SyncLogs{}).@@ -103,38 +112,41 @@ func (d *SyncTaskDAO) ListDueTasks(ctx context.Context, now time.Time, limit intJoins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").Where("sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).Order("sync_logs.update_time DESC").- Limit(limit * 4).+ Limit(limit).Scan(&rows).Error; err != nil {return nil, err}-- tasks := make([]entity.SyncLogs, 0, limit)+ tasks := make([]ScheduledSyncTask, 0, len(rows))for _, row := range rows {- if !isDue(row.SyncLogs, row.ConnectorRefreshFreq, row.ConnectorPruneFreq, row.ConnectorConfig, now) {- continue- }- tasks = append(tasks, row.SyncLogs)- if len(tasks) >= limit {- break- }+ tasks = append(tasks, ScheduledSyncTask{+ SyncLogs: row.SyncLogs,+ ConnectorRefreshFreq: row.ConnectorRefreshFreq,+ ConnectorPruneFreq: row.ConnectorPruneFreq,+ ConnectorConfig: row.ConnectorConfig,+ })}return tasks, nil}-// ListStartupTasks returns scheduled tasks that should be published during NATS startup reconciliation.-func (d *SyncTaskDAO) ListStartupTasks(ctx context.Context, limit int) ([]entity.SyncLogs, error) {- var tasks []entity.SyncLogs- err := d.db.WithContext(ctx).+// GetScheduledTask returns one scheduled task with connector scheduling settings.+func (d *SyncTaskDAO) GetScheduledTask(ctx context.Context, taskID string) (ScheduledSyncTask, error) {+ var row dueSyncTaskRow+ if err := d.db.WithContext(ctx).Model(&entity.SyncLogs{}).- Select("sync_logs.*").+ Select("sync_logs.*, connector.refresh_freq AS connector_refresh_freq, connector.prune_freq AS connector_prune_freq, connector.config AS connector_config").Joins("JOIN connector ON sync_logs.connector_id = connector.id").Joins("JOIN connector2kb ON sync_logs.connector_id = connector2kb.connector_id AND sync_logs.kb_id = connector2kb.kb_id").Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").- Where("sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).- Order("sync_logs.update_time DESC").- Limit(limit).- Find(&tasks).Error- return tasks, err+ Where("sync_logs.id = ? AND sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", taskID, SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).+ First(&row).Error; err != nil {+ return ScheduledSyncTask{}, err+ }+ return ScheduledSyncTask{+ SyncLogs: row.SyncLogs,+ ConnectorRefreshFreq: row.ConnectorRefreshFreq,+ ConnectorPruneFreq: row.ConnectorPruneFreq,+ ConnectorConfig: row.ConnectorConfig,+ }, nil}// ClaimTask conditionally marks a scheduled task as running.@@ -268,9 +280,55 @@ func (d *SyncTaskDAO) FailTask(ctx context.Context, taskID, connectorID, message})}+// HandleTransientFailure retries a running task until maxRetries is reached.+func (d *SyncTaskDAO) HandleTransientFailure(ctx context.Context, taskID, connectorID, message string, maxRetries int64) (int64, bool, error) {+ var attempts int64+ var failed bool+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var task entity.SyncLogs+ query := tx.WithContext(ctx)+ if tx.Dialector.Name() != "sqlite" {+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})+ }+ if err := query.Where("id = ? AND status = ?", taskID, SyncStatusRunning).First(&task).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil+ }+ return err+ }++ attempts = task.ErrorCount + 1+ status := SyncStatusSchedule+ connectorStatus := SyncStatusSchedule+ errorMsg := message+ if attempts >= maxRetries {+ failed = true+ status = SyncStatusFail+ connectorStatus = SyncStatusFail+ errorMsg = fmt.Sprintf("sync task failed after %d transient retries: %s", maxRetries, message)+ }++ if err := tx.Model(&entity.SyncLogs{}).+ Where("id = ? AND status = ?", taskID, SyncStatusRunning).+ Updates(map[string]any{+ "status": status,+ "error_msg": errorMsg,+ "error_count": attempts,+ }).Error; err != nil {+ return err+ }+ if connectorID == "" {+ return nil+ }+ return tx.Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", connectorStatus).Error+ })+ return attempts, failed, err+}+// CompleteSyncTask marks SYNC done and creates the next schedule task.-func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, newDocs, totalDocs, errorCount int64, errorMsg string) error {- return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, newDocs, totalDocs, errorCount int64, errorMsg string) (string, error) {+ var nextTaskID string+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {result := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", taskContext.Task.ID, SyncStatusRunning).Updates(map[string]any{"status": SyncStatusDone,"poll_range_end": pollRangeEnd,@@ -289,13 +347,17 @@ func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskreturn err}- return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypeSync, false, &pollRangeEnd, taskContext.Task.TotalDocsIndexed+totalDocs)+ var err error+ nextTaskID, err = createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypeSync, false, &pollRangeEnd, taskContext.Task.TotalDocsIndexed+totalDocs)+ return err})+ return nextTaskID, err}// CompletePruneTask marks PRUNE done and creates the next schedule task.-func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTaskContext, removed int64) error {- return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTaskContext, removed int64) (string, error) {+ var nextTaskID string+ err := d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {result := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", taskContext.Task.ID, SyncStatusRunning).Updates(map[string]any{"status": SyncStatusDone,"docs_removed_from_index": gorm.Expr("docs_removed_from_index + ?", removed),@@ -312,8 +374,11 @@ func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTasif !syncConnectorConfigBool(taskContext.Connector.Config, "sync_deleted_files") {return nil}- return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypePrune, false, nil, taskContext.Task.TotalDocsIndexed)+ var err error+ nextTaskID, err = createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypePrune, false, nil, taskContext.Task.TotalDocsIndexed)+ return err})+ return nextTaskID, err}// RecoverStaleRunning restores timed-out running tasks to schedule.@@ -387,10 +452,15 @@ func (d *SyncTaskDAO) RecoverRunning(ctx context.Context) (int64, error) {connectorIDs := map[string]struct{}{}return int64(len(rows)), d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ dueAt := time.Unix(0, 0).Local()for _, row := range rows {if err := tx.Model(&entity.SyncLogs{}).Where("id = ? AND status = ?", row.ID, SyncStatusRunning).- Update("status", SyncStatusSchedule).Error; err != nil {+ Updates(map[string]any{+ "status": SyncStatusSchedule,+ "update_time": dueAt.UnixMilli(),+ "update_date": dueAt,+ }).Error; err != nil {return err}connectorIDs[row.ConnectorID] = struct{}{}@@ -405,7 +475,7 @@ func (d *SyncTaskDAO) RecoverRunning(ctx context.Context) (int64, error) {}// createScheduledTask creates the next Python-compatible scheduled task.-func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taskType string, fromBeginning bool, pollRangeStart *time.Time, totalDocsIndexed int64) error {+func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taskType string, fromBeginning bool, pollRangeStart *time.Time, totalDocsIndexed int64) (string, error) {var lockRow entity.Connector2Kbquery := tx.WithContext(ctx)if tx.Dialector.Name() != "sqlite" {@@ -414,17 +484,19 @@ func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taif err := query.Where("connector_id = ? AND kb_id = ?", connectorID, kbID).First(&lockRow).Error; err != nil {- return err+ return "", err}- var existing int64- if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).+ var existing entity.SyncLogs+ err := tx.WithContext(ctx).Where("connector_id = ? AND kb_id = ? AND task_type = ? AND status = ?", connectorID, kbID, taskType, SyncStatusSchedule).- Count(&existing).Error; err != nil {- return err+ Order("update_time DESC").+ First(&existing).Error+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {+ return "", err}- if existing > 0 {- return nil+ if err == nil {+ return existing.ID, nil}reindex := "0"@@ -436,10 +508,11 @@ func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taif err := tx.WithContext(ctx).Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusSchedule).Error; err != nil {- return err+ return "", err}- return tx.WithContext(ctx).Create(&entity.SyncLogs{- ID: utility.GenerateToken(),+ taskID := utility.GenerateToken()+ return taskID, tx.WithContext(ctx).Create(&entity.SyncLogs{+ ID: taskID,ConnectorID: connectorID,KbID: kbID,TaskType: taskType,@@ -452,29 +525,6 @@ func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, ta}).Error}-// isDue applies refresh_freq and prune_freq scheduling semantics.-func isDue(task entity.SyncLogs, refreshFreq, pruneFreq int64, config map[string]any, now time.Time) bool {- if task.UpdateDate == nil {- return true- }- var freqMinutes int64- switch task.TaskType {- case TaskTypeSync:- freqMinutes = refreshFreq- case TaskTypePrune:- if !syncConnectorConfigBool(config, "sync_deleted_files") {- return false- }- freqMinutes = pruneFreq- default:- return false- }- if freqMinutes <= 0 {- return true- }- return task.UpdateDate.Before(now.Add(-time.Duration(freqMinutes) * time.Minute))-}-// syncConnectorConfigBool reads a Python JSON bool/string flag.func syncConnectorConfigBool(config map[string]any, key string) bool {value, ok := config[key]diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex d809699898..6bab124d74 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -44,9 +44,10 @@ type NatsEngine struct {knowledgeCompileConsumer jetstream.Consumerkv jetstream.KeyValue- syncerStream jetstream.Stream- syncerConsumer jetstream.Consumer- syncerMu sync.Mutex+ syncerStream jetstream.Stream+ syncerConsumer jetstream.PushConsumer+ syncCheckpointKV jetstream.KeyValue+ syncerMu sync.Mutex}func NewNatsEngine(host string, port int) *NatsEngine {diff --git a/internal/engine/nats/syncer.go b/internal/engine/nats/syncer.goindex 5b109d5171..83cf0179df 100644--- a/internal/engine/nats/syncer.go+++ b/internal/engine/nats/syncer.go@@ -25,9 +25,10 @@ import ("time""ragflow/internal/common"+ syncerconnector "ragflow/internal/syncer/connector"- "github.com/nats-io/nats.go""github.com/nats-io/nats.go/jetstream"+ "go.uber.org/zap")const (@@ -36,7 +37,11 @@ const (syncerStreamName = "RAGFLOW_SYNC_TASKS"syncerConsumerName = "RAGFLOW_SYNCER_CONSUMER"+ syncCheckpointBucket = "RAGFLOW_SYNC_CHECKPOINTS"+ syncerDeliverSubject = "deliver.syncer.RAGFLOW"+ syncerDeliverGroup = "RAGFLOW_SYNCER_WORKERS"syncerSubjectPattern = "sync.tasks.>"+ syncCheckpointTTL = 7 * 24 * time.Hour)// InitSyncerStream creates the datasource syncer task stream.@@ -79,7 +84,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {return nil}-// InitSyncerConsumer creates the durable pull consumer for syncer tasks.+// InitSyncerConsumer creates the durable push consumer for syncer tasks.func (n *NatsEngine) InitSyncerConsumer() error {n.syncerMu.Lock()defer n.syncerMu.Unlock()@@ -89,28 +94,109 @@ func (n *NatsEngine) InitSyncerConsumer() error {ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()- consumer, err := n.syncerStream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{- Name: syncerConsumerName,- Durable: syncerConsumerName,- AckPolicy: jetstream.AckExplicitPolicy,- MaxDeliver: 16,- MaxAckPending: 1024 * 128,- FilterSubject: syncerSubjectPattern,- })- if err != nil {- if strings.Contains(err.Error(), "max waiting can not be updated") {- consumer, err = n.syncerStream.Consumer(ctx, syncerConsumerName)- if err != nil {- return fmt.Errorf("syncer: get existing consumer: %w", err)+ config := syncerPushConsumerConfig()+ consumer, err := n.syncerStream.CreateOrUpdatePushConsumer(ctx, config)+ if err != nil && shouldRecreateSyncerConsumer(ctx, n.syncerStream, config, err) {+ common.Warn("syncer replacing incompatible NATS consumer", zap.String("consumer", syncerConsumerName), zap.Error(err), zap.Bool("delete", true))+ if deleteErr := n.syncerStream.DeleteConsumer(ctx, syncerConsumerName); deleteErr != nil {+ if !strings.Contains(strings.ToLower(deleteErr.Error()), "not found") {+ return fmt.Errorf("syncer: replace existing consumer: %w", deleteErr)}- } else {- return fmt.Errorf("syncer: create consumer: %w", err)}+ consumer, err = n.syncerStream.CreateOrUpdatePushConsumer(ctx, config)+ }+ if err != nil {+ return fmt.Errorf("syncer: create push consumer: %w", err)}n.syncerConsumer = consumerreturn nil}+// InitSyncCheckpoints creates the KV bucket backing running sync task checkpoints.+func (n *NatsEngine) InitSyncCheckpoints() error {+ n.syncerMu.Lock()+ defer n.syncerMu.Unlock()+ return n.initSyncCheckpointsLocked()+}++func (n *NatsEngine) initSyncCheckpointsLocked() error {+ if n.jetStream == nil {+ return fmt.Errorf("syncer checkpoint: jetStream not initialized")+ }+ if n.syncCheckpointKV != nil {+ return nil+ }++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ kv, err := n.jetStream.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{+ Bucket: syncCheckpointBucket,+ Description: "running datasource sync checkpoints",+ History: 1,+ TTL: syncCheckpointTTL,+ MaxValueSize: 1024 * 1024,+ Storage: jetstream.FileStorage,+ })+ if err != nil {+ return fmt.Errorf("syncer checkpoint: create kv: %w", err)+ }+ n.syncCheckpointKV = kv+ return nil+}++func syncerPushConsumerConfig() jetstream.ConsumerConfig {+ return jetstream.ConsumerConfig{+ Name: syncerConsumerName,+ Durable: syncerConsumerName,+ AckPolicy: jetstream.AckExplicitPolicy,+ MaxDeliver: 16,+ MaxAckPending: 1024,+ FilterSubject: syncerSubjectPattern,+ DeliverSubject: syncerDeliverSubject,+ DeliverGroup: syncerDeliverGroup,+ }+}++func shouldRecreateSyncerConsumer(ctx context.Context, stream jetstream.Stream, desired jetstream.ConsumerConfig, err error) bool {+ var jsErr jetstream.JetStreamError+ if !errors.As(err, &jsErr) || jsErr.APIError() == nil || jsErr.APIError().ErrorCode != jetstream.JSErrCodeConsumerCreate {+ return false+ }++ existing, inspectErr := syncerExistingConsumerConfig(ctx, stream)+ if inspectErr != nil {+ common.Warn("syncer consumer replacement skipped", zap.String("consumer", syncerConsumerName), zap.Error(err), zap.NamedError("inspect_error", inspectErr), zap.Bool("delete", false))+ return false+ }+ if existing == nil {+ common.Warn("syncer consumer replacement skipped", zap.String("consumer", syncerConsumerName), zap.Error(err), zap.Bool("delete", false))+ return false+ }+ return syncerConsumerConfigMismatch(*existing, desired)+}++func syncerExistingConsumerConfig(ctx context.Context, stream jetstream.Stream) (*jetstream.ConsumerConfig, error) {+ lister := stream.ListConsumers(ctx)+ for info := range lister.Info() {+ if info != nil && info.Name == syncerConsumerName {+ config := info.Config+ return &config, nil+ }+ }+ return nil, lister.Err()+}++func syncerConsumerConfigMismatch(existing, desired jetstream.ConsumerConfig) bool {+ return existing.Durable != desired.Durable ||+ existing.Name != desired.Name ||+ existing.AckPolicy != desired.AckPolicy ||+ existing.MaxDeliver != desired.MaxDeliver ||+ existing.MaxAckPending != desired.MaxAckPending ||+ existing.FilterSubject != desired.FilterSubject ||+ existing.DeliverSubject != desired.DeliverSubject ||+ existing.DeliverGroup != desired.DeliverGroup+}+// PublishSyncerTask publishes one sync_logs task wake-up.func (n *NatsEngine) PublishSyncerTask(taskID string) error {if err := n.InitSyncerStream(); err != nil {@@ -129,29 +215,116 @@ func (n *NatsEngine) PublishSyncerTask(taskID string) error {return err}-// FetchSyncerTasks pulls syncer task messages from JetStream.-func (n *NatsEngine) FetchSyncerTasks(batchSize int) ([]common.TaskHandle, error) {+// PublishSyncerTaskWakeup publishes a non-deduplicated wake-up for an existing sync_logs task.+func (n *NatsEngine) PublishSyncerTaskWakeup(taskID string) error {+ if err := n.InitSyncerStream(); err != nil {+ return err+ }++ payload, err := json.Marshal(common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeSyncer})+ if err != nil {+ return err+ }++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ msgID := fmt.Sprintf("%s.manual-wakeup.%d", taskID, time.Now().UnixNano())+ _, err = n.jetStream.Publish(ctx, SyncerTaskSubject, payload, jetstream.WithMsgID(msgID), jetstream.WithExpectStream(syncerStreamName))+ return err+}++// SubscribeSyncerTasks starts push delivery for syncer task messages.+func (n *NatsEngine) SubscribeSyncerTasks(ctx context.Context, handler func(common.TaskHandle)) error {n.syncerMu.Lock()consumer := n.syncerConsumern.syncerMu.Unlock()+if consumer == nil {- return nil, fmt.Errorf("syncer: consumer not initialized")+ return fmt.Errorf("syncer: consumer not initialized")+ }++ consumeCtx, err := consumer.Consume(func(msg jetstream.Msg) {+ handler(NewNatsMessageHandle(msg))+ })+ if err != nil {+ return err+ }++ go func() {+ <-ctx.Done()+ consumeCtx.Stop()+ <-consumeCtx.Closed()+ }()+ return nil+}++// LoadSyncCheckpoint reads the latest running checkpoint for one sync task.+func (n *NatsEngine) LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error) {+ kv, err := n.syncCheckpointStore()+ if err != nil {+ return nil, err}- // fetch task from nats(jetStream)- messages, err := consumer.Fetch(batchSize, jetstream.FetchMaxWait(1*time.Second))++ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)+ defer cancel()+ entry, err := kv.Get(ctx, syncCheckpointKey(taskID))if err != nil {- if errors.Is(err, nats.ErrTimeout) {+ if errors.Is(err, jetstream.ErrKeyNotFound) {return nil, nil}return nil, err}- handles := make([]common.TaskHandle, 0, batchSize)- for msg := range messages.Messages() {- handles = append(handles, NewNatsMessageHandle(msg))+ var state syncerconnector.SyncCheckpointState+ if err = json.Unmarshal(entry.Value(), &state); err != nil {+ return nil, err}- if err = messages.Error(); err != nil {- return handles, err+ return &state, nil+}++// SaveSyncCheckpoint writes the latest running checkpoint for one sync task.+func (n *NatsEngine) SaveSyncCheckpoint(ctx context.Context, taskID string, state syncerconnector.SyncCheckpointState) error {+ kv, err := n.syncCheckpointStore()+ if err != nil {+ return err}- return handles, nil++ // write to json+ data, err := json.Marshal(state)+ if err != nil {+ return err+ }+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)+ defer cancel()+ _, err = kv.Put(ctx, syncCheckpointKey(taskID), data)+ return err+}++// DeleteSyncCheckpoint removes the running checkpoint for a completed sync task.+func (n *NatsEngine) DeleteSyncCheckpoint(ctx context.Context, taskID string) error {+ kv, err := n.syncCheckpointStore()+ if err != nil {+ return err+ }++ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)+ defer cancel()+ err = kv.Delete(ctx, syncCheckpointKey(taskID))+ if errors.Is(err, jetstream.ErrKeyNotFound) {+ return nil+ }+ return err+}++func (n *NatsEngine) syncCheckpointStore() (jetstream.KeyValue, error) {+ n.syncerMu.Lock()+ defer n.syncerMu.Unlock()+ if err := n.initSyncCheckpointsLocked(); err != nil {+ return nil, err+ }+ return n.syncCheckpointKV, nil+}++func syncCheckpointKey(taskID string) string {+ return "task." + taskID}diff --git a/internal/engine/nats/syncer_test.go b/internal/engine/nats/syncer_test.goindex 03ae16adbc..5ec96d6a10 100644--- a/internal/engine/nats/syncer_test.go+++ b/internal/engine/nats/syncer_test.go@@ -1,7 +1,9 @@package natsimport (+ "context""net"+ syncerconnector "ragflow/internal/syncer/connector""testing""time"@@ -10,8 +12,8 @@ import ("github.com/nats-io/nats-server/v2/server")-// TestSyncerTaskStreamPublishesAndFetches verifies the dedicated syncer stream path.-func TestSyncerTaskStreamPublishesAndFetches(t *testing.T) {+// TestSyncerTaskStreamPublishesAndSubscribes verifies the dedicated syncer stream path.+func TestSyncerTaskStreamPublishesAndSubscribes(t *testing.T) {engine := setupSyncerNATSEngine(t)if err := engine.InitSyncerStream(); err != nil {t.Fatalf("InitSyncerStream: %v", err)@@ -19,26 +21,73 @@ func TestSyncerTaskStreamPublishesAndFetches(t *testing.T) {if err := engine.InitSyncerConsumer(); err != nil {t.Fatalf("InitSyncerConsumer: %v", err)}+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ received := make(chan common.TaskHandle, 1)+ if err := engine.SubscribeSyncerTasks(ctx, func(handle common.TaskHandle) {+ received <- handle+ }); err != nil {+ t.Fatalf("SubscribeSyncerTasks: %v", err)+ }if err := engine.PublishSyncerTask("task-1"); err != nil {t.Fatalf("PublishSyncerTask: %v", err)}- handles, err := engine.FetchSyncerTasks(1)- if err != nil {- t.Fatalf("FetchSyncerTasks: %v", err)- }- if len(handles) != 1 {- t.Fatalf("handles len = %d, want 1", len(handles))+ var handle common.TaskHandle+ select {+ case handle = <-received:+ case <-time.After(5 * time.Second):+ t.Fatal("timed out waiting for syncer task")}- message := handles[0].GetMessage()+ message := handle.GetMessage()if message.TaskID != "task-1" || message.TaskType != common.TaskTypeSyncer {t.Fatalf("message = %+v", message)}- if err := handles[0].Ack(); err != nil {+ if err := handle.Ack(); err != nil {t.Fatalf("Ack: %v", err)}}+// TestSyncCheckpointKVSaveLoadDelete verifies the syncer checkpoint KV lifecycle.+func TestSyncCheckpointKVSaveLoadDelete(t *testing.T) {+ engine := setupSyncerNATSEngine(t)+ now := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)+ state := syncerconnector.SyncCheckpointState{+ Version: 1,+ TaskID: "task-1",+ ConnectorID: "conn-1",+ KBID: "kb-1",+ WindowEnd: now,+ NextCommitSeq: 2,+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &now, SourceID: "source-1"},+ }+ if err := engine.SaveSyncCheckpoint(t.Context(), "task-1", state); err != nil {+ t.Fatalf("SaveSyncCheckpoint: %v", err)+ }++ loaded, err := engine.LoadSyncCheckpoint(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("LoadSyncCheckpoint: %v", err)+ }+ if loaded == nil || loaded.Checkpoint == nil {+ t.Fatalf("loaded checkpoint = %+v, want value", loaded)+ }+ if loaded.TaskID != "task-1" || loaded.Checkpoint.Cursor != "cursor-1" || loaded.Checkpoint.SourceID != "source-1" {+ t.Fatalf("loaded checkpoint = %+v", loaded)+ }++ if err = engine.DeleteSyncCheckpoint(t.Context(), "task-1"); err != nil {+ t.Fatalf("DeleteSyncCheckpoint: %v", err)+ }+ loaded, err = engine.LoadSyncCheckpoint(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("LoadSyncCheckpoint after delete: %v", err)+ }+ if loaded != nil {+ t.Fatalf("loaded checkpoint after delete = %+v, want nil", loaded)+ }+}+func setupSyncerNATSEngine(t *testing.T) *NatsEngine {t.Helper()opts := &server.Options{diff --git a/internal/handler/connector.go b/internal/handler/connector.goindex ce1aac2bcc..d71e22ef00 100644--- a/internal/handler/connector.go+++ b/internal/handler/connector.go@@ -38,6 +38,7 @@ type connectorServiceIface interface {ListLog(ctx context.Context, connectorID, userID string, page, pageSize int) ([]*entity.ConnectorSyncLog, int64, common.ErrorCode, error)DeleteConnector(ctx context.Context, connectorID, userID string) (bool, common.ErrorCode, error)RebuildConnector(ctx context.Context, connectorID, userID, kbID string) (bool, common.ErrorCode, error)+ ResumeFailedSync(ctx context.Context, connectorID, userID string, req *service.ResumeFailedSyncRequest) (bool, common.ErrorCode, error)TestConnector(ctx context.Context, connectorID, userID string) errorUpdateConnector(ctx context.Context, connectorID, userID string, req *service.UpdateConnectorRequest) (*entity.Connector, common.ErrorCode, error)StartGoogleWebOAuth(ctx context.Context, userID, source string, req *service.StartGoogleWebOAuthRequest) (*service.StartGoogleWebOAuthResponse, common.ErrorCode, error)@@ -380,6 +381,36 @@ func (h *ConnectorHandler) RebuildConnector(c *gin.Context) {common.SuccessWithData(c, ok, "success")}+// ResumeFailedSync resumes a failed connector sync task from checkpoint.+// @Summary Resume Failed Connector Sync+// @Description Resume a failed connector sync task from its saved checkpoint+// @Tags connector+// @Accept json+// @Produce json+// @Success 200 {object} map[string]interface{}+// @Router /connector/:connector_id/resume-failed-sync [post]+func (h *ConnectorHandler) ResumeFailedSync(c *gin.Context) {+ user, errorCode, errorMessage := GetUser(c)+ if errorCode != common.CodeSuccess {+ common.ErrorWithCode(c, errorCode, errorMessage)+ return+ }++ var req service.ResumeFailedSyncRequest+ if err := c.ShouldBindJSON(&req); err != nil {+ common.ResponseWithCodeData(c, common.CodeDataError, nil, "required argument is missing: kb_id or task_id")+ return+ }++ ctx := c.Request.Context()+ ok, code, err := h.connectorService.ResumeFailedSync(ctx, c.Param("connector_id"), user.ID, &req)+ if err != nil {+ common.ErrorWithCode(c, code, err.Error())+ return+ }+ common.SuccessWithData(c, ok, "success")+}+func (h *ConnectorHandler) StartGoogleWebOAuth(c *gin.Context) {user, errorCode, errorMessage := GetUser(c)if errorCode != common.CodeSuccess {diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.goindex 676b9411c7..57080390b4 100644--- a/internal/handler/connector_test.go+++ b/internal/handler/connector_test.go@@ -119,6 +119,13 @@ func (s fakeConnectorService) RebuildConnector(context.Context, string, string,return true, common.CodeSuccess, nil}+func (s fakeConnectorService) ResumeFailedSync(context.Context, string, string, *service.ResumeFailedSyncRequest) (bool, common.ErrorCode, error) {+ if s.err != nil {+ return false, s.code, s.err+ }+ return true, common.CodeSuccess, nil+}+func TestConnectorHandlerStartBoxWebOAuth(t *testing.T) {gin.SetMode(gin.TestMode)diff --git a/internal/router/router.go b/internal/router/router.goindex fd2dc699c2..0dc12816c4 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -635,6 +635,7 @@ func (r *Router) Setup(engine *gin.Engine) {connectors.GET("/:connector_id/logs", r.connectorHandler.ListLogs)connectors.DELETE("/:connector_id", r.connectorHandler.DeleteConnector)connectors.POST("/:connector_id/rebuild", r.connectorHandler.RebuildConnector)+ connectors.POST("/:connector_id/resume-failed-sync", r.connectorHandler.ResumeFailedSync)connectors.POST("/:connector_id/test", r.connectorHandler.TestConnector)}diff --git a/internal/server/config/syncer_config.go b/internal/server/config/syncer_config.goindex 16f1911de4..4265242bb5 100644--- a/internal/server/config/syncer_config.go+++ b/internal/server/config/syncer_config.go@@ -20,13 +20,11 @@ import "github.com/spf13/viper"type SyncerConfig struct {MaxConcurrentSyncs int `mapstructure:"max_concurrent_syncs"`- SyncInterval int `mapstructure:"sync_interval"`}func (c *Config) ParseSyncerConfig(v *viper.Viper) error {// Default Syncer configc.syncer.MaxConcurrentSyncs = 5- c.syncer.SyncInterval = 3if !v.IsSet("file_syncer") {return nil@@ -40,10 +38,6 @@ func (c *Config) ParseSyncerConfig(v *viper.Viper) error {c.syncer.MaxConcurrentSyncs = sub.GetInt("max_concurrent_syncs")}- if sub.IsSet("sync_interval") {- c.syncer.SyncInterval = sub.GetInt("sync_interval")- }-return nil}diff --git a/internal/service/connector.go b/internal/service/connector.goindex 17fabc2149..e3fd4133a1 100644--- a/internal/service/connector.go+++ b/internal/service/connector.go@@ -29,10 +29,12 @@ import ("net/http""net/url""ragflow/internal/engine/redis"+ syncerconnector "ragflow/internal/syncer/connector""ragflow/internal/utility""strings""time"+ "go.uber.org/zap""gorm.io/gorm""ragflow/internal/common"@@ -91,11 +93,33 @@ type syncTaskPublisher interface {PublishSyncerTask(taskID string) error}+type syncTaskWakeupPublisher interface {+ PublishSyncerTaskWakeup(taskID string) error+}++type syncCheckpointLoader interface {+ LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error)+}++type syncCheckpointDeleter interface {+ DeleteSyncCheckpoint(ctx context.Context, taskID string) error+}+var getSyncerTaskPublisher = func() (syncTaskPublisher, bool) {publisher, ok := engine.GetMessageQueueEngine().(syncTaskPublisher)return publisher, ok}+var getSyncCheckpointLoader = func() (syncCheckpointLoader, bool) {+ loader, ok := engine.GetMessageQueueEngine().(syncCheckpointLoader)+ return loader, ok+}++var getSyncCheckpointDeleter = func() (syncCheckpointDeleter, bool) {+ deleter, ok := engine.GetMessageQueueEngine().(syncCheckpointDeleter)+ return deleter, ok+}+// NewConnectorService create connector servicefunc NewConnectorService() *ConnectorService {return &ConnectorService{@@ -124,6 +148,12 @@ type RebuildConnectorRequest struct {KbID string `json:"kb_id"`}+// ResumeFailedSyncRequest resumes a failed connector sync task from checkpoint.+type ResumeFailedSyncRequest struct {+ KbID string `json:"kb_id"`+ TaskID string `json:"task_id"`+}+type StartGoogleWebOAuthRequest struct {Credentials json.RawMessage `json:"credentials"`RedirectURI string `json:"redirect_uri,omitempty"`@@ -934,9 +964,7 @@ func (s *ConnectorService) UpdateConnector(ctx context.Context, connectorID, useif err != nil {return nil, common.CodeServerError, err}- if err = publishSyncerTasks(taskIDs); err != nil {- return nil, common.CodeServerError, err- }+ publishSyncerTasks(taskIDs)} else if isConnectorCancelStatus(req.Status) {if err = s.cancelConnectorTasks(ctx, connectorID); err != nil {return nil, common.CodeServerError, err@@ -946,9 +974,7 @@ func (s *ConnectorService) UpdateConnector(ctx context.Context, connectorID, useif err != nil {return nil, common.CodeServerError, err}- if err = publishSyncerTasks(taskIDs); err != nil {- return nil, common.CodeServerError, err- }+ publishSyncerTasks(taskIDs)}}@@ -1006,29 +1032,127 @@ func (s *ConnectorService) RebuildConnector(ctx context.Context, connectorID, uss.deleteConnectorDocumentChunks(ctx, connector.TenantID, kbID, documents)- taskIDs, err := s.connectorDAO.RebuildConnector(ctx, dao.DB, connector, kbID, documents)+ taskIDs, oldSyncTaskIDs, err := s.connectorDAO.RebuildConnector(ctx, dao.DB, connector, kbID, documents)+ if err != nil {+ return false, common.CodeServerError, err+ }++ if err = deleteSyncCheckpoints(ctx, oldSyncTaskIDs); err != nil {+ common.Warn("delete sync checkpoints failed during rebuild",+ zap.String("connector_id", connectorID), zap.Error(err))+ }++ publishSyncerTasks(taskIDs)+ return true, common.CodeSuccess, nil+}++// ResumeFailedSync schedules a failed sync task to continue from its saved checkpoint.+func (s *ConnectorService) ResumeFailedSync(ctx context.Context, connectorID, userID string, req *ResumeFailedSyncRequest) (bool, common.ErrorCode, error) {+ if connectorID == "" {+ return false, common.CodeDataError, fmt.Errorf("connector_id is required")+ }+ if req == nil {+ return false, common.CodeArgumentError, fmt.Errorf("request is required")+ }+ // check KBid and TaskID+ req.KbID = strings.TrimSpace(req.KbID)+ req.TaskID = strings.TrimSpace(req.TaskID)+ if req.KbID == "" {+ return false, common.CodeArgumentError, fmt.Errorf("required argument is missing: kb_id")+ }+ if req.TaskID == "" {+ return false, common.CodeArgumentError, fmt.Errorf("required argument is missing: task_id")+ }+ // get connector+ connector, err := s.connectorDAO.GetByID(ctx, dao.DB, connectorID)+ if err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, common.CodeDataError, fmt.Errorf("can't find this Connector")+ }+ return false, common.CodeServerError, err+ }+ // check access+ canAccess, err := s.canAccessConnector(ctx, connector, userID)+ if err != nil {+ return false, common.CodeServerError, err+ }+ if !canAccess {+ return false, common.CodeAuthenticationError, fmt.Errorf("no authorization")+ }++ // load the checkpoint+ loader, ok := getSyncCheckpointLoader()+ if !ok {+ return false, common.CodeServerError, fmt.Errorf("sync checkpoint store is not configured")+ }++ checkpoint, err := loader.LoadSyncCheckpoint(ctx, req.TaskID)if err != nil {return false, common.CodeServerError, err}- if err = publishSyncerTasks(taskIDs); err != nil {+ if checkpoint == nil || checkpoint.Checkpoint == nil {+ return false, common.CodeDataError, fmt.Errorf("checkpoint not found, cannot resume failed sync task")+ }+ if checkpoint.TaskID != req.TaskID || checkpoint.ConnectorID != connectorID || checkpoint.KBID != req.KbID {+ return false, common.CodeDataError, fmt.Errorf("checkpoint does not match connector, knowledge base, and task")+ }+ // resume the task+ if err = s.connectorDAO.ResumeFailedSyncTask(ctx, dao.DB, connectorID, req.KbID, req.TaskID); err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, common.CodeDataError, fmt.Errorf("failed sync task not found")+ }+ return false, common.CodeServerError, err+ }+ if err = publishSyncerTaskWakeup(req.TaskID); err != nil {return false, common.CodeServerError, err}return true, common.CodeSuccess, nil}-func publishSyncerTasks(taskIDs []string) error {+func publishSyncerTasks(taskIDs []string) {if len(taskIDs) == 0 {- return nil+ return}publisher, ok := getSyncerTaskPublisher()if !ok {- return fmt.Errorf("syncer task publisher is not configured")+ common.Warn("syncer task publisher is not configured")+ return}for _, taskID := range taskIDs {if taskID == "" {continue}if err := publisher.PublishSyncerTask(taskID); err != nil {+ common.Warn("syncer task publish failed", zap.String("task_id", taskID), zap.Error(err))+ }+ }+}++// publishSyncerTaskWakeup publish syncer task wakeup message to nats+func publishSyncerTaskWakeup(taskID string) error {+ publisher, ok := getSyncerTaskPublisher()+ if !ok {+ return fmt.Errorf("syncer task publisher is not configured")+ }+ if wakeupPublisher, ok := publisher.(syncTaskWakeupPublisher); ok {+ return wakeupPublisher.PublishSyncerTaskWakeup(taskID)+ }+ return publisher.PublishSyncerTask(taskID)+}++func deleteSyncCheckpoints(ctx context.Context, taskIDs []string) error {+ if len(taskIDs) == 0 {+ return nil+ }+ deleter, ok := getSyncCheckpointDeleter()+ if !ok {+ return fmt.Errorf("sync checkpoint store is not configured")+ }+ for _, taskID := range taskIDs {+ if taskID == "" {+ continue+ }+ if err := deleter.DeleteSyncCheckpoint(ctx, taskID); err != nil {return err}}diff --git a/internal/service/connector_syncer_test.go b/internal/service/connector_syncer_test.goindex a29824830f..2f70859500 100644--- a/internal/service/connector_syncer_test.go+++ b/internal/service/connector_syncer_test.go@@ -7,10 +7,12 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ syncerconnector "ragflow/internal/syncer/connector")type fakeSyncerTaskPublisher struct {- taskIDs []string+ taskIDs []string+ wakeupTaskIDs []string}func (p *fakeSyncerTaskPublisher) PublishSyncerTask(taskID string) error {@@ -18,6 +20,29 @@ func (p *fakeSyncerTaskPublisher) PublishSyncerTask(taskID string) error {return nil}+func (p *fakeSyncerTaskPublisher) PublishSyncerTaskWakeup(taskID string) error {+ p.wakeupTaskIDs = append(p.wakeupTaskIDs, taskID)+ return nil+}++type fakeSyncCheckpointLoader struct {+ state *syncerconnector.SyncCheckpointState+ deletedIDs []string+}++func (l *fakeSyncCheckpointLoader) LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error) {+ return l.state, nil+}++func (l *fakeSyncCheckpointLoader) DeleteSyncCheckpoint(ctx context.Context, taskID string) error {+ l.deletedIDs = append(l.deletedIDs, taskID)+ return nil+}++func stringPtr(value string) *string {+ return &value+}+func TestUpdateConnectorSchedulePublishesSyncerTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -86,6 +111,166 @@ func TestUpdateConnectorSchedulePublishesSyncerTask(t *testing.T) {}}+func TestResumeFailedSyncSchedulesOriginalTaskFromCheckpoint(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ if err := db.AutoMigrate(&entity.Connector{}, &entity.SyncLogs{}); err != nil {+ t.Fatalf("migrate connector tables: %v", err)+ }+ if err := db.Create(&entity.Connector{+ ID: "conn-1",+ TenantID: "user-1",+ Name: "conn-1",+ Source: "gmail",+ InputType: "poll",+ Config: entity.JSONMap{},+ Status: string(entity.TaskStatusFail),+ RefreshFreq: 0,+ PruneFreq: 0,+ TimeoutSecs: 60,+ }).Error; err != nil {+ t.Fatalf("insert connector: %v", err)+ }+ if err := db.Create(&entity.SyncLogs{+ ID: "task-1",+ ConnectorID: "conn-1",+ KbID: "kb-1",+ TaskType: dao.TaskTypeSync,+ Status: string(entity.TaskStatusFail),+ FromBeginning: stringPtr("1"),+ ErrorMsg: "sync task failed after 3 transient retries: unexpected EOF",+ ErrorCount: 3,+ }).Error; err != nil {+ t.Fatalf("insert sync log: %v", err)+ }++ publisher := &fakeSyncerTaskPublisher{}+ previousPublisher := getSyncerTaskPublisher+ getSyncerTaskPublisher = func() (syncTaskPublisher, bool) {+ return publisher, true+ }+ previousLoader := getSyncCheckpointLoader+ store := &fakeSyncCheckpointLoader{state: &syncerconnector.SyncCheckpointState{+ Version: 1,+ TaskID: "task-1",+ ConnectorID: "conn-1",+ KBID: "kb-1",+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", SourceID: "source-1"},+ }}+ getSyncCheckpointLoader = func() (syncCheckpointLoader, bool) {+ return store, true+ }+ t.Cleanup(func() {+ getSyncerTaskPublisher = previousPublisher+ getSyncCheckpointLoader = previousLoader+ })++ ok, code, err := NewConnectorService().ResumeFailedSync(context.Background(), "conn-1", "user-1", &ResumeFailedSyncRequest{KbID: "kb-1", TaskID: "task-1"})+ if err != nil {+ t.Fatalf("ResumeFailedSync error: %v", err)+ }+ if !ok || code != common.CodeSuccess {+ t.Fatalf("ok/code = %v/%v, want true/success", ok, code)+ }+ if len(publisher.wakeupTaskIDs) != 1 || publisher.wakeupTaskIDs[0] != "task-1" {+ t.Fatalf("wakeup task IDs = %v, want task-1", publisher.wakeupTaskIDs)+ }+ if len(publisher.taskIDs) != 0 {+ t.Fatalf("regular publish task IDs = %v, want none", publisher.taskIDs)+ }++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != string(entity.TaskStatusSchedule) {+ t.Fatalf("task status = %s, want schedule", task.Status)+ }+ if task.ErrorCount != 0 || task.ErrorMsg != "" {+ t.Fatalf("error count/msg = %d/%q, want reset", task.ErrorCount, task.ErrorMsg)+ }+ if task.FromBeginning == nil || *task.FromBeginning != "1" {+ t.Fatalf("from_beginning = %v, want preserved full-sync task", task.FromBeginning)+ }+}++func TestRebuildConnectorDeletesOldSyncCheckpointsBeforePublishing(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ if err := db.AutoMigrate(&entity.Connector{}, &entity.Connector2Kb{}, &entity.Knowledgebase{}, &entity.SyncLogs{}); err != nil {+ t.Fatalf("migrate connector tables: %v", err)+ }+ if err := db.Create(&entity.Connector{+ ID: "conn-1",+ TenantID: "user-1",+ Name: "conn-1",+ Source: "gmail",+ InputType: "poll",+ Config: entity.JSONMap{},+ Status: string(entity.TaskStatusFail),+ RefreshFreq: 0,+ PruneFreq: 0,+ TimeoutSecs: 60,+ }).Error; err != nil {+ t.Fatalf("insert connector: %v", err)+ }+ if err := db.Create(&entity.Knowledgebase{+ ID: "kb-1",+ TenantID: "user-1",+ Name: "kb-1",+ CreatedBy: "user-1",+ EmbdID: "embd",+ }).Error; err != nil {+ t.Fatalf("insert kb: %v", err)+ }+ if err := db.Create(&entity.Connector2Kb{ID: "conn-1-kb-1", ConnectorID: "conn-1", KbID: "kb-1", AutoParse: "1"}).Error; err != nil {+ t.Fatalf("insert connector mapping: %v", err)+ }+ if err := db.Create([]entity.SyncLogs{+ {ID: "failed-sync", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusFail), ErrorMsg: ""},+ {ID: "done-sync", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusDone), ErrorMsg: ""},+ {ID: "failed-prune", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypePrune, Status: string(entity.TaskStatusFail), ErrorMsg: ""},+ }).Error; err != nil {+ t.Fatalf("insert sync logs: %v", err)+ }++ publisher := &fakeSyncerTaskPublisher{}+ previousPublisher := getSyncerTaskPublisher+ getSyncerTaskPublisher = func() (syncTaskPublisher, bool) {+ return publisher, true+ }+ store := &fakeSyncCheckpointLoader{}+ previousDeleter := getSyncCheckpointDeleter+ getSyncCheckpointDeleter = func() (syncCheckpointDeleter, bool) {+ return store, true+ }+ t.Cleanup(func() {+ getSyncerTaskPublisher = previousPublisher+ getSyncCheckpointDeleter = previousDeleter+ })++ ok, code, err := NewConnectorService().RebuildConnector(context.Background(), "conn-1", "user-1", "kb-1")+ if err != nil {+ t.Fatalf("RebuildConnector error: %v", err)+ }+ if !ok || code != common.CodeSuccess {+ t.Fatalf("ok/code = %v/%v, want true/success", ok, code)+ }+ if len(publisher.taskIDs) != 1 {+ t.Fatalf("published task IDs = %v, want one new rebuild task", publisher.taskIDs)+ }+ if len(store.deletedIDs) != 2 {+ t.Fatalf("deleted checkpoint IDs = %v, want failed-sync and done-sync", store.deletedIDs)+ }+ deleted := map[string]bool{}+ for _, taskID := range store.deletedIDs {+ deleted[taskID] = true+ }+ if !deleted["failed-sync"] || !deleted["done-sync"] || deleted["failed-prune"] {+ t.Fatalf("deleted checkpoint IDs = %v", store.deletedIDs)+ }+}+func TestUpdateConnectorScheduleDoesNotDuplicateRunningTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.goindex 2cd95310d5..0a2cdbee07 100644--- a/internal/service/dataset/update.go+++ b/internal/service/dataset/update.go@@ -9,6 +9,7 @@ import ("ragflow/internal/common""ragflow/internal/dao"+ "ragflow/internal/engine""ragflow/internal/engine/types""ragflow/internal/entity"pipelinepkg "ragflow/internal/ingestion/pipeline"@@ -170,6 +171,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantIDtxCode := common.CodeSuccessvar updatedKB *entity.Knowledgebasevar linkedConnectors []*dao.ConnectorDatasetListItem+ var scheduledTaskIDs []stringvar pagerankUpdate *datasetPagerankUpdateerr = dao.DB.Transaction(func(tx *gorm.DB) error {lockedKB, code, authErr := d.lockAccessibleDatasetForUpdate(tx, datasetID, tenantID)@@ -320,7 +322,8 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID}if connectorsProvided {- if err = d.connectorDAO.LinkDatasetConnectorsTx(ctx, tx, lockedKB.ID, lockedKB.TenantID, connectorLinks); err != nil {+ scheduledTaskIDs, err = d.connectorDAO.LinkDatasetConnectorsTx(ctx, tx, lockedKB.ID, lockedKB.TenantID, connectorLinks)+ if err != nil {if dao.IsConnectorNotAccessibleErr(err) {txCode = common.CodeDataErrorreturn err@@ -350,6 +353,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID}return nil, txCode, err}+ publishDatasetSyncerTasks(scheduledTaskIDs)if pagerankUpdate != nil {if err = d.updateDatasetPagerankChunks(ctx, *pagerankUpdate); err != nil {@@ -379,6 +383,29 @@ func (d *DatasetService) updateDatasetPagerankChunks(ctx context.Context, updatereturn err}+type datasetSyncTaskPublisher interface {+ PublishSyncerTask(taskID string) error+}++func publishDatasetSyncerTasks(taskIDs []string) {+ if len(taskIDs) == 0 {+ return+ }+ publisher, ok := engine.GetMessageQueueEngine().(datasetSyncTaskPublisher)+ if !ok {+ common.Warn("syncer task publisher is not configured")+ return+ }+ for _, taskID := range taskIDs {+ if taskID == "" {+ continue+ }+ if err := publisher.PublishSyncerTask(taskID); err != nil {+ common.Warn("syncer task publish failed", zap.String("task_id", taskID), zap.Error(err))+ }+ }+}+func (d *DatasetService) lockAccessibleDatasetForUpdate(tx *gorm.DB, datasetID, userID string) (*entity.Knowledgebase, common.ErrorCode, error) {var kb entity.Knowledgebaseerr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).diff --git a/internal/service/sync_document_id_resolver.go b/internal/service/sync_document_id_resolver.goindex b9605badc7..68edbf66b4 100644--- a/internal/service/sync_document_id_resolver.go+++ b/internal/service/sync_document_id_resolver.go@@ -42,6 +42,7 @@ type ResolvedDocumentID struct {// DocumentStore reads existing connector documents.type DocumentStore interface {ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error)+ ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error)GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error)}@@ -55,6 +56,11 @@ func NewDocumentIDResolver(store DocumentStore) *DocumentIDResolver {return &DocumentIDResolver{store: store}}+// ListFingerprintsBySourceType returns all fingerprints for one connector namespace.+func (r *DocumentIDResolver) ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error) {+ return r.store.ListFingerprintsBySourceType(ctx, kbID, sourceType)+}+// Resolve chooses the legacy ID when already present in the KB.func (r *DocumentIDResolver) Resolve(ctx context.Context, kbID, connectorID, sourceType, sourceID string) (ResolvedDocumentID, error) {legacyID := Hash128(connectorID + ":" + sourceID)@@ -100,6 +106,26 @@ func (s *GormDocumentStore) ListIDs(ctx context.Context, kbID, sourceType stringreturn result, nil}+// ListFingerprintsBySourceType returns all existing fingerprints for one connector namespace.+func (s *GormDocumentStore) ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error) {+ var docs []entity.Document+ if err := dao.GetDB().WithContext(ctx).+ Select("id", "content_hash").+ Where("kb_id = ? AND source_type = ?", kbID, sourceType).+ Find(&docs).Error; err != nil {+ return nil, err+ }+ result := make(map[string]string, len(docs))+ for _, doc := range docs {+ if doc.ContentHash == nil {+ result[doc.ID] = ""+ continue+ }+ result[doc.ID] = *doc.ContentHash+ }+ return result, nil+}+// GetFingerprintsByIDs returns fingerprints for the requested candidate IDs.func (s *GormDocumentStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {var docs []entity.Documentdiff --git a/internal/service/sync_task_service.go b/internal/service/sync_task_service.goindex b3cf3e8064..83fcc26370 100644--- a/internal/service/sync_task_service.go+++ b/internal/service/sync_task_service.go@@ -34,6 +34,9 @@ const (// SyncTaskContext contains the database rows needed to execute a task.type SyncTaskContext = dao.SyncTaskContext+// ScheduledSyncTask contains one scheduled task and its connector schedule settings.+type ScheduledSyncTask = dao.ScheduledSyncTask+// SyncStats accumulates task-level document results.type SyncStats struct {Added int64@@ -79,30 +82,14 @@ func NewSyncTaskService(taskDAO *dao.SyncTaskDAO) *SyncTaskService {return &SyncTaskService{taskDAO: taskDAO}}-// ListDueTasks returns due schedule tasks.-func (s *SyncTaskService) ListDueTasks(ctx context.Context, now time.Time) ([]dao.SyncTask, error) {- tasks, err := s.taskDAO.ListDueTasks(ctx, now, 128)- if err != nil {- return nil, err- }- out := make([]dao.SyncTask, 0, len(tasks))- for _, task := range tasks {- out = append(out, dao.SyncTask{SyncLogs: task})- }- return out, nil+// ListScheduledTasks returns scheduled tasks for one-time timer reconciliation.+func (s *SyncTaskService) ListScheduledTasks(ctx context.Context) ([]ScheduledSyncTask, error) {+ return s.taskDAO.ListScheduledTasks(ctx, 4096)}-// ListStartupTasks returns scheduled tasks for NATS startup reconciliation.-func (s *SyncTaskService) ListStartupTasks(ctx context.Context) ([]dao.SyncTask, error) {- tasks, err := s.taskDAO.ListStartupTasks(ctx, 128)- if err != nil {- return nil, err- }- out := make([]dao.SyncTask, 0, len(tasks))- for _, task := range tasks {- out = append(out, dao.SyncTask{SyncLogs: task})- }- return out, nil+// GetScheduledTask returns one scheduled task for timer registration.+func (s *SyncTaskService) GetScheduledTask(ctx context.Context, taskID string) (ScheduledSyncTask, error) {+ return s.taskDAO.GetScheduledTask(ctx, taskID)}// Claim marks a scheduled task running if no other scanner claimed it first.@@ -144,14 +131,23 @@ func (s *SyncTaskService) Fail(ctx context.Context, taskID, connectorID string,return s.taskDAO.FailTask(ctx, taskID, connectorID, message, 1)}+// HandleTransientFailure retries a running task until maxRetries is reached.+func (s *SyncTaskService) HandleTransientFailure(ctx context.Context, taskID, connectorID string, err error, maxRetries int64) (int64, bool, error) {+ message := ""+ if err != nil {+ message = err.Error()+ }+ return s.taskDAO.HandleTransientFailure(ctx, taskID, connectorID, message, maxRetries)+}+// CompleteSync commits a successful SYNC task and schedules the next one.-func (s *SyncTaskService) CompleteSync(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, stats SyncStats) error {+func (s *SyncTaskService) CompleteSync(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, stats SyncStats) (string, error) {changed := stats.Added + stats.Updatedreturn s.taskDAO.CompleteSyncTask(ctx, taskContext, pollRangeEnd, changed, changed, stats.ErrorCount, stats.ErrorMsg)}// CompletePrune commits a successful PRUNE task and schedules the next one.-func (s *SyncTaskService) CompletePrune(ctx context.Context, taskContext SyncTaskContext, removed int64) error {+func (s *SyncTaskService) CompletePrune(ctx context.Context, taskContext SyncTaskContext, removed int64) (string, error) {return s.taskDAO.CompletePruneTask(ctx, taskContext, removed)}diff --git a/internal/syncer/checkpoint_store.go b/internal/syncer/checkpoint_store.gonew file mode 100644index 0000000000..018f3c5200--- /dev/null+++ b/internal/syncer/checkpoint_store.go@@ -0,0 +1,91 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ syncerconnector "ragflow/internal/syncer/connector"+ "sync"+)++// SyncCheckpointStore persists running sync task checkpoints.+type SyncCheckpointStore interface {+ LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error)+ SaveSyncCheckpoint(ctx context.Context, taskID string, state syncerconnector.SyncCheckpointState) error+ DeleteSyncCheckpoint(ctx context.Context, taskID string) error+}++type memorySyncCheckpointStore struct {+ mu sync.Mutex+ states map[string]syncerconnector.SyncCheckpointState+}++func newMemorySyncCheckpointStore() *memorySyncCheckpointStore {+ return &memorySyncCheckpointStore{states: map[string]syncerconnector.SyncCheckpointState{}}+}++func cloneSyncCheckpointState(state syncerconnector.SyncCheckpointState) syncerconnector.SyncCheckpointState {+ clone := state+ if state.WindowStart != nil {+ windowStart := *state.WindowStart+ clone.WindowStart = &windowStart+ }++ if state.Checkpoint != nil {+ checkpoint := *state.Checkpoint+ if checkpoint.UpdatedAt != nil {+ updatedAt := *checkpoint.UpdatedAt+ checkpoint.UpdatedAt = &updatedAt+ }+ clone.Checkpoint = &checkpoint+ }+ return clone+}++func (s *memorySyncCheckpointStore) LoadSyncCheckpoint(ctx context.Context, taskID string) (*syncerconnector.SyncCheckpointState, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ state, ok := s.states[taskID]+ if !ok {+ return nil, nil+ }+ clone := cloneSyncCheckpointState(state)+ return &clone, nil+}++func (s *memorySyncCheckpointStore) SaveSyncCheckpoint(ctx context.Context, taskID string, state syncerconnector.SyncCheckpointState) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ s.states[taskID] = cloneSyncCheckpointState(state)+ return nil+}++func (s *memorySyncCheckpointStore) DeleteSyncCheckpoint(ctx context.Context, taskID string) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ delete(s.states, taskID)+ return nil+}diff --git a/internal/syncer/config.go b/internal/syncer/config.goindex 2b9604b0e6..a14da4ca7c 100644--- a/internal/syncer/config.go+++ b/internal/syncer/config.go@@ -22,7 +22,6 @@ import (// Config contains runtime limits for the datasource syncer.type Config struct {- PollInterval time.DurationTaskWorkerCount intTaskQueueSize intJobWorkerCount int@@ -34,7 +33,6 @@ type Config struct {// DefaultConfig returns the first-version syncer defaults.func DefaultConfig() Config {return Config{- PollInterval: 3 * time.Second,TaskWorkerCount: 5,TaskQueueSize: 10,JobWorkerCount: 400,@@ -47,10 +45,6 @@ func DefaultConfig() Config {// Normalize fills unset or invalid config values with defaults.func (c Config) Normalize() Config {def := DefaultConfig()- if c.PollInterval <= 0 {- c.PollInterval = def.PollInterval- }-if c.TaskWorkerCount <= 0 {c.TaskWorkerCount = def.TaskWorkerCount}diff --git a/internal/syncer/connector/fingerprint.go b/internal/syncer/connector/fingerprint.goindex b463cca42b..6c86506703 100644--- a/internal/syncer/connector/fingerprint.go+++ b/internal/syncer/connector/fingerprint.go@@ -20,15 +20,55 @@ import ("encoding/hex""encoding/json""fmt"+ "path/filepath"+ "regexp"+ "strings""github.com/zeebo/xxh3")+var syncFilenameUnsafeRE = regexp.MustCompile(`[\\/:*?"<>|]+`)+func stableFingerprint(value any) string {data, err := json.Marshal(value)if err != nil {data = []byte(fmt.Sprint(value))}+ return contentFingerprint(data)+}++func contentFingerprint(data []byte) string {sum := xxh3.Hash128(data).Bytes()return hex.EncodeToString(sum[:])}++func syncSourceDocumentFilenameFromDocument(doc SourceDocument) string {+ name := strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(doc.SemanticIdentifier, "_"))+ if name == "" {+ name = strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(doc.SourceID, "_"))+ }+ if name == "" {+ name = "document"+ }++ extension := strings.TrimSpace(doc.Extension)+ if extension == "" {+ extension = ".txt"+ }+ if !strings.HasPrefix(extension, ".") {+ extension = "." + extension+ }+ if !strings.HasSuffix(strings.ToLower(name), strings.ToLower(extension)) {+ name += extension+ }+ if len(name) <= 255 {+ return name+ }++ ext := filepath.Ext(name)+ baseLimit := 255 - len(ext)+ if baseLimit < 1 {+ return name[:255]+ }+ return name[:baseLimit] + ext+}diff --git a/internal/syncer/connector/gmail.go b/internal/syncer/connector/gmail.goindex 9f18f6f5c0..1380c5ffcd 100644--- a/internal/syncer/connector/gmail.go+++ b/internal/syncer/connector/gmail.go@@ -100,7 +100,14 @@ func (c *GmailConnector) OpenSync(ctx context.Context, request SyncRequest) (Synif !request.FromBeginning && request.WindowStart != nil {query = gmailTimeRangeQuery(request.WindowStart, request.WindowEnd)}- return &gmailSyncSession{connector: c, users: users, batchSize: c.batchSize, query: query}, nil+ session := &gmailSyncSession{+ connector: c,+ users: users,+ batchSize: c.batchSize,+ query: query,+ }+ session.applyResume(request.Resume)+ return session, nil}// OpenPrune opens one complete Gmail prune snapshot session.@@ -288,21 +295,28 @@ func (c *GmailConnector) getJSON(ctx context.Context, client *http.Client, apiUR}type gmailSyncSession struct {- connector *GmailConnector- users []string- userIndex int- pageToken string- batchSize int- query string- buffer []SourceDocument+ connector *GmailConnector+ users []string+ userIndex int+ pageToken string+ batchSize int+ query string+ buffer []gmailBufferedDocument+ resumePageToken string+ resumeOffset int+ resumeFilename string}// NextBatch returns the next Gmail document batch.func (s *gmailSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpointif len(s.buffer) > 0 {n := min(s.batchSize, len(s.buffer))- documents = append(documents, s.buffer[:n]...)+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }s.buffer = s.buffer[n:]}for len(documents) < s.batchSize {@@ -318,13 +332,19 @@ func (s *gmailSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {}remaining := s.batchSize - len(documents)if len(batch) > remaining {- documents = append(documents, batch[:remaining]...)+ for _, buffered := range batch[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }s.buffer = append(s.buffer, batch[remaining:]...)break}- documents = append(documents, batch...)+ for _, buffered := range batch {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }}- return SyncBatch{Documents: documents}, nil+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil}// Close closes the Gmail sync session.@@ -333,9 +353,10 @@ func (s *gmailSyncSession) Close() error {}// nextDocumentPage fetches one Gmail list page and expands threads.-func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {+func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]gmailBufferedDocument, error) {userEmail := s.users[s.userIndex]- page, err := s.connector.listThreads(ctx, userEmail, s.query, s.pageToken, gmailItemsPerPage)+ requestPageToken := s.pageToken+ page, err := s.connector.listThreads(ctx, userEmail, s.query, requestPageToken, gmailItemsPerPage)if err != nil {if isGmailDisabled(err) {s.advanceUser()@@ -343,7 +364,8 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocume}return nil, err}- documents := make([]SourceDocument, 0, len(page.Threads))+ candidates := make([]gmailBufferedDocument, 0, len(page.Threads))+ pageOffset := 0for _, item := range page.Threads {thread, err := s.connector.loadThread(ctx, userEmail, item.ID)if err != nil {@@ -354,9 +376,16 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocume}doc, ok := thread.toSourceDocument(userEmail)if ok {- documents = append(documents, doc)+ pageOffset+++ candidates = append(candidates, gmailBufferedDocument{+ document: doc,+ checkpoint: gmailSyncCheckpoint(userEmail, requestPageToken, pageOffset, doc),+ offset: pageOffset,+ filename: syncSourceDocumentFilenameFromDocument(doc),+ })}}+ documents := s.filterResumedDocuments(requestPageToken, candidates)if page.NextPageToken == "" {s.advanceUser()} else {@@ -365,10 +394,103 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocumereturn documents, nil}+// applyResume apply resume if resume is available (adjust session's token and pageOffset)+func (s *gmailSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }++ var cursor gmailSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return+ }+ if cursor.UserEmail == "" {+ return+ }+ for index, userEmail := range s.users {+ if userEmail != cursor.UserEmail {+ continue+ }+ s.userIndex = index+ s.pageToken = cursor.PageToken+ s.resumePageToken = cursor.PageToken+ s.resumeFilename = cursor.Filename+ if cursor.Offset > 0 {+ s.resumeOffset = cursor.Offset+ }+ return+ }+}++// filterResumedDocuments applies checkpoint offset using the filename at that offset.+func (s *gmailSyncSession) filterResumedDocuments(pageToken string, candidates []gmailBufferedDocument) []gmailBufferedDocument {+ if s.resumeOffset <= 0 {+ return candidates+ }+ if pageToken != s.resumePageToken {+ s.clearResumeOffset()+ return candidates+ }+ if s.resumeOffset <= len(candidates) && candidates[s.resumeOffset-1].filename == s.resumeFilename {+ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset > s.resumeOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResumeOffset()+ return filtered+ }++ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset >= s.resumeOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResumeOffset()+ return filtered+}++func (s *gmailSyncSession) clearResumeOffset() {+ s.resumeOffset = 0+ s.resumePageToken = ""+ s.resumeFilename = ""+}+// advanceUser moves a Gmail session to the next mailbox.func (s *gmailSyncSession) advanceUser() {s.userIndex++s.pageToken = ""+ s.clearResumeOffset()+}++type gmailBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+ offset int+ filename string+}++type gmailSyncCursor struct {+ UserEmail string `json:"user_email"`+ PageToken string `json:"page_token,omitempty"`+ Offset int `json:"offset"`+ Filename string `json:"filename"`+}++func gmailSyncCheckpoint(userEmail, pageToken string, offset int, doc SourceDocument) *SyncCheckpoint {+ filename := syncSourceDocumentFilenameFromDocument(doc)+ cursor, err := json.Marshal(gmailSyncCursor{UserEmail: userEmail, PageToken: pageToken, Offset: offset, Filename: filename})+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: string(cursor),+ UpdatedAt: &updatedAt,+ SourceID: doc.SourceID,+ }}type gmailPruneSession struct {@@ -523,12 +645,7 @@ func (t gmailThread) toSourceDocument(userEmail string) (SourceDocument, bool) {UpdatedAt: updatedAt,SizeBytes: int64(len(blob)),Metadata: metadata,- Fingerprint: stableFingerprint(map[string]any{- "thread_id": t.ID,- "updated_at": updatedAt,- "blob": string(blob),- "metadata": metadata,- }),+ Fingerprint: contentFingerprint(blob),}, true}diff --git a/internal/syncer/connector/gmail_test.go b/internal/syncer/connector/gmail_test.goindex 6bfa76f96f..22c3f05e16 100644--- a/internal/syncer/connector/gmail_test.go+++ b/internal/syncer/connector/gmail_test.go@@ -59,6 +59,100 @@ func TestGmailConnectorOpenSync(t *testing.T) {}}+// TestGmailConnectorOpenSyncResumesWithinPage verifies Gmail checkpoint resumes inside a list page.+func TestGmailConnectorOpenSyncResumesWithinPage(t *testing.T) {+ connector := newFixtureGmailConnector()+ connector.GmailConnector.batchSize = 2+ connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {+ return gmailThreadListPage{Threads: []struct {+ ID string `json:"id"`+ }{{ID: "thread-1"}, {ID: "thread-2"}, {ID: "thread-3"}}}, nil+ }+ connector.GmailConnector.getThread = func(ctx context.Context, userEmail, threadID string) (gmailThread, error) {+ return gmailTestThread(threadID, "Fri, 02 Jan 2026 03:04:05 +0000")+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if len(first.Documents) != 2 {+ t.Fatalf("first documents len = %d, want 2", len(first.Documents))+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "thread-2" {+ t.Fatalf("first checkpoint = %+v, want thread-2", first.Checkpoint)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "thread-3" {+ t.Fatalf("resume documents = %+v, want thread-3", second.Documents)+ }+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "thread-3" {+ t.Fatalf("resume checkpoint = %+v, want thread-3", second.Checkpoint)+ }+}++// TestGmailConnectorResumeUsesFilenameWhenOffsetChanged verifies page deletions resume from the current filename offset.+func TestGmailConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {+ connector := newFixtureGmailConnector()+ connector.GmailConnector.batchSize = 2+ connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {+ return gmailThreadListPage{Threads: []struct {+ ID string `json:"id"`+ }{{ID: "thread-1"}, {ID: "thread-2"}, {ID: "thread-3"}}}, nil+ }+ connector.GmailConnector.getThread = func(ctx context.Context, userEmail, threadID string) (gmailThread, error) {+ return gmailTestThreadWithSubject(threadID, threadID, "Fri, 02 Jan 2026 03:04:05 +0000")+ }+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "thread-2" {+ t.Fatalf("first checkpoint = %+v, want thread-2", first.Checkpoint)+ }++ resumeListCalls := 0+ connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {+ resumeListCalls+++ if pageToken != "" {+ t.Fatalf("resume used unexpected pageToken=%q", pageToken)+ }+ return gmailThreadListPage{Threads: []struct {+ ID string `json:"id"`+ }{{ID: "thread-1"}, {ID: "thread-3"}}}, nil+ }+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "thread-3" {+ t.Fatalf("resume documents = %+v, want thread-3", second.Documents)+ }+ if resumeListCalls != 1 {+ t.Fatalf("resume list calls = %d, want one checkpoint page", resumeListCalls)+ }+}+// TestGmailFingerprintStable verifies Gmail fingerprints are stable and content-sensitive.func TestGmailFingerprintStable(t *testing.T) {thread := gmailThread{@@ -91,6 +185,9 @@ func TestGmailFingerprintStable(t *testing.T) {if doc1.Fingerprint == "" || doc1.Fingerprint != doc2.Fingerprint {t.Fatalf("fingerprint unstable: %q %q", doc1.Fingerprint, doc2.Fingerprint)}+ if doc1.Fingerprint != contentFingerprint(doc1.Blob) {+ t.Fatalf("fingerprint = %q, want content fingerprint", doc1.Fingerprint)+ }changed := threadchanged.Messages = append([]gmailMessage(nil), thread.Messages...)@@ -191,28 +288,36 @@ func newFixtureGmailConnector() *fixtureGmailConnector {}{{ID: "thread-1"}}}, nil}base.getThread = func(ctx context.Context, userEmail, threadID string) (gmailThread, error) {- return gmailThread{- ID: threadID,- Messages: []gmailMessage{{- ID: "msg-1",- Payload: gmailPayload{- Headers: []gmailHeader{- {Name: "From", Value: "Alice Example <[email protected]>"},- {Name: "To", Value: "Bob <[email protected]>"},- {Name: "Subject", Value: "Hello/World"},- {Name: "Date", Value: "Fri, 02 Jan 2026 03:04:05 +0000"},- },- Parts: []gmailPart{{- MimeType: "text/plain",- Body: gmailBody{Data: base64.RawURLEncoding.EncodeToString([]byte("Body text"))},- }},- },- }},- }, nil+ return gmailTestThread(threadID, "Fri, 02 Jan 2026 03:04:05 +0000")}return connector}+func gmailTestThread(threadID, date string) (gmailThread, error) {+ return gmailTestThreadWithSubject(threadID, "Hello/World", date)+}++func gmailTestThreadWithSubject(threadID, subject, date string) (gmailThread, error) {+ return gmailThread{+ ID: threadID,+ Messages: []gmailMessage{{+ ID: "msg-1",+ Payload: gmailPayload{+ Headers: []gmailHeader{+ {Name: "From", Value: "Alice Example <[email protected]>"},+ {Name: "To", Value: "Bob <[email protected]>"},+ {Name: "Subject", Value: subject},+ {Name: "Date", Value: date},+ },+ Parts: []gmailPart{{+ MimeType: "text/plain",+ Body: gmailBody{Data: base64.RawURLEncoding.EncodeToString([]byte("Body text"))},+ }},+ },+ }},+ }, nil+}+// TestGmailTimeRangeQuery verifies Python-compatible after and before seconds.func TestGmailTimeRangeQuery(t *testing.T) {start := time.Unix(100, 0).UTC()diff --git a/internal/syncer/connector/google_drive.go b/internal/syncer/connector/google_drive.goindex 96da7d9113..cef4f821b6 100644--- a/internal/syncer/connector/google_drive.go+++ b/internal/syncer/connector/google_drive.go@@ -38,7 +38,7 @@ import (const (defaultGoogleDriveBatchSize = 32- defaultGoogleDriveSizeThreshold = 10 * 1024 * 1024+ defaultGoogleDriveSizeThreshold = 64 * 1024 * 1024googleDriveItemsPerPage = 100googleDriveRequestTimeout = 60 * time.SecondgoogleDriveOAuthTokenURL = "https://oauth2.googleapis.com/token"@@ -144,14 +144,15 @@ func (c *GoogleDriveConnector) OpenSync(ctx context.Context, request SyncRequestif err != nil {return nil, err}- return &googleDriveSyncSession{+ session := &googleDriveSyncSession{connector: c,scopes: scopes,scopeIndex: 0,batchSize: c.batchSize,windowStart: request.WindowStart,windowEnd: request.WindowEnd,- }, nil+ }+ return session.applyResume(request.Resume), nil}// OpenPrune opens one complete Google Drive prune snapshot session.@@ -540,16 +541,19 @@ func (c *GoogleDriveConnector) getBytes(ctx context.Context, client *http.Client}type googleDriveSyncSession struct {- connector *GoogleDriveConnector- scopes []googleDriveScope- scopeIndex int- pageToken string- batchSize int- windowStart *time.Time- windowEnd time.Time- buffer []SourceDocument- seen map[string]struct{}- folderSeen map[string]struct{}+ connector *GoogleDriveConnector+ scopes []googleDriveScope+ scopeIndex int+ pageToken string+ batchSize int+ windowStart *time.Time+ windowEnd time.Time+ buffer []googleDriveBufferedDocument+ seen map[string]struct{}+ folderSeen map[string]struct{}+ resumePageToken string+ resumePageOffset int+ resumeFilename string}// NextBatch returns the next Google Drive document batch.@@ -561,9 +565,13 @@ func (s *googleDriveSyncSession) NextBatch(ctx context.Context) (SyncBatch, erros.folderSeen = map[string]struct{}{}}documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpointif len(s.buffer) > 0 {n := min(s.batchSize, len(s.buffer))- documents = append(documents, s.buffer[:n]...)+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }s.buffer = s.buffer[n:]}for len(documents) < s.batchSize {@@ -579,18 +587,25 @@ func (s *googleDriveSyncSession) NextBatch(ctx context.Context) (SyncBatch, erro}remaining := s.batchSize - len(documents)if len(batch) > remaining {- documents = append(documents, batch[:remaining]...)+ for _, buffered := range batch[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }s.buffer = append(s.buffer, batch[remaining:]...)break}- documents = append(documents, batch...)+ for _, buffered := range batch {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }}- return SyncBatch{Documents: documents}, nil+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil}-func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {+func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]googleDriveBufferedDocument, error) {scope := s.scopes[s.scopeIndex]- page, err := s.connector.listFilePageWithRetry(ctx, scope, s.pageToken, s.windowStart, s.windowEnd)+ requestPageToken := s.pageToken+ page, err := s.connector.listFilePageWithRetry(ctx, scope, requestPageToken, s.windowStart, s.windowEnd)if err != nil {if isGooglePermissionDeniedOrNotFound(err) {s.advanceScope()@@ -598,18 +613,22 @@ func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]Source}return nil, err}- docs := make([]SourceDocument, 0, len(page.Files))+ candidates := make([]googleDriveBufferedDocument, 0, len(page.Files))+ pageOffset := 0for _, file := range page.Files {doc, ok := file.toSourceDocument(scope.userEmail, s.connector.allowImages)if !ok {continue}- if _, ok = s.seen[doc.SourceID]; ok {- continue- }- s.seen[doc.SourceID] = struct{}{}- docs = append(docs, doc)- }+ pageOffset+++ candidates = append(candidates, googleDriveBufferedDocument{+ document: doc,+ checkpoint: s.syncCheckpoint(scope, requestPageToken, pageOffset, doc),+ offset: pageOffset,+ filename: syncSourceDocumentFilenameFromDocument(doc),+ })+ }+ docs := s.filterSeenDocuments(s.filterResumedDocuments(requestPageToken, candidates))if page.NextPageToken == "" {if err = s.finishScope(ctx, scope); err != nil {return nil, err@@ -643,6 +662,7 @@ func (s *googleDriveSyncSession) finishScope(ctx context.Context, scope googleDrfunc (s *googleDriveSyncSession) advanceScope() {s.scopeIndex++s.pageToken = ""+ s.clearResumeOffset()}// Close closes the Google Drive sync session.@@ -655,6 +675,185 @@ func (s *googleDriveSyncSession) Fetch(ctx context.Context, ref FetchReference)return s.connector.Fetch(ctx, ref)}+func (s *googleDriveSyncSession) filterSeenDocuments(candidates []googleDriveBufferedDocument) []googleDriveBufferedDocument {+ filtered := candidates[:0]+ for _, candidate := range candidates {+ if _, ok := s.seen[candidate.document.SourceID]; ok {+ continue+ }+ s.seen[candidate.document.SourceID] = struct{}{}+ filtered = append(filtered, candidate)+ }+ return filtered+}++// applyResume apply resume if checkpoint is available+func (s *googleDriveSyncSession) applyResume(checkpoint *SyncCheckpoint) *googleDriveSyncSession {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return s+ }+ var cursor googleDriveSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return s+ }+ if len(cursor.Scopes) > 0 && cursor.ScopeIndex >= 0 && cursor.ScopeIndex < len(cursor.Scopes) {+ scopes := make([]googleDriveScope, 0, len(cursor.Scopes))+ for _, scope := range cursor.Scopes {+ scopes = append(scopes, scope.toScope())+ }+ s.scopes = scopes+ s.scopeIndex = cursor.ScopeIndex+ s.markFolderScopesSeen()+ } else if scope, ok := cursor.Scope.toScopeIfValid(); ok {+ if index := googleDriveScopeIndex(s.scopes, scope); index >= 0 {+ s.scopeIndex = index+ } else {+ s.scopes = append(s.scopes, scope)+ s.scopeIndex = len(s.scopes) - 1+ }+ }+ s.pageToken = cursor.PageToken+ s.resumePageToken = cursor.PageToken+ s.resumeFilename = cursor.Filename+ if cursor.Offset > 0 {+ s.resumePageOffset = cursor.Offset+ }+ return s+}++func (s *googleDriveSyncSession) filterResumedDocuments(pageToken string, candidates []googleDriveBufferedDocument) []googleDriveBufferedDocument {+ if s.resumePageOffset <= 0 {+ return candidates+ }+ if pageToken != s.resumePageToken {+ s.clearResumeOffset()+ return candidates+ }+ if s.resumePageOffset <= len(candidates) && candidates[s.resumePageOffset-1].filename == s.resumeFilename {+ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset > s.resumePageOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResumeOffset()+ return filtered+ }++ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset >= s.resumePageOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResumeOffset()+ return filtered+}++func (s *googleDriveSyncSession) clearResumeOffset() {+ s.resumePageOffset = 0+ s.resumePageToken = ""+ s.resumeFilename = ""+}++func (s *googleDriveSyncSession) syncCheckpoint(scope googleDriveScope, pageToken string, offset int, doc SourceDocument) *SyncCheckpoint {+ scopes := make([]googleDriveScopeCursor, 0, len(s.scopes))+ for _, scope := range s.scopes {+ scopes = append(scopes, googleDriveScopeCursorFromScope(scope))+ }+ cursor, err := json.Marshal(googleDriveSyncCursor{+ Scopes: scopes,+ Scope: googleDriveScopeCursorFromScope(scope),+ ScopeIndex: s.scopeIndex,+ PageToken: pageToken,+ Offset: offset,+ Filename: syncSourceDocumentFilenameFromDocument(doc),+ })+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: string(cursor),+ UpdatedAt: &updatedAt,+ SourceID: doc.SourceID,+ }+}++func (s *googleDriveSyncSession) markFolderScopesSeen() {+ if s.folderSeen == nil {+ s.folderSeen = map[string]struct{}{}+ }+ for _, scope := range s.scopes {+ if scope.corpora == "folder" && scope.folderID != "" {+ s.folderSeen[scope.folderID] = struct{}{}+ }+ }+}++type googleDriveBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+ offset int+ filename string+}++type googleDriveSyncCursor struct {+ Scopes []googleDriveScopeCursor `json:"scopes,omitempty"`+ Scope googleDriveScopeCursor `json:"scope"`+ ScopeIndex int `json:"scope_index"`+ PageToken string `json:"page_token,omitempty"`+ Offset int `json:"offset"`+ Filename string `json:"filename"`+}++type googleDriveScopeCursor struct {+ UserEmail string `json:"user_email"`+ Corpora string `json:"corpora"`+ DriveID string `json:"drive_id,omitempty"`+ FolderID string `json:"folder_id,omitempty"`+ IncludeSharedWithMe bool `json:"include_shared_with_me,omitempty"`+ SharedWithMeOnly bool `json:"shared_with_me_only,omitempty"`+}++func googleDriveScopeCursorFromScope(scope googleDriveScope) googleDriveScopeCursor {+ return googleDriveScopeCursor{+ UserEmail: scope.userEmail,+ Corpora: scope.corpora,+ DriveID: scope.driveID,+ FolderID: scope.folderID,+ IncludeSharedWithMe: scope.includeSharedWithMe,+ SharedWithMeOnly: scope.sharedWithMeOnly,+ }+}++func (c googleDriveScopeCursor) toScope() googleDriveScope {+ return googleDriveScope{+ userEmail: c.UserEmail,+ corpora: c.Corpora,+ driveID: c.DriveID,+ folderID: c.FolderID,+ includeSharedWithMe: c.IncludeSharedWithMe,+ sharedWithMeOnly: c.SharedWithMeOnly,+ }+}++func (c googleDriveScopeCursor) toScopeIfValid() (googleDriveScope, bool) {+ if c.UserEmail == "" || c.Corpora == "" {+ return googleDriveScope{}, false+ }+ return c.toScope(), true+}++func googleDriveScopeIndex(scopes []googleDriveScope, target googleDriveScope) int {+ for index, scope := range scopes {+ if scope == target {+ return index+ }+ }+ return -1+}+type googleDrivePruneSession struct {connector *GoogleDriveConnectorscopes []googleDriveScopediff --git a/internal/syncer/connector/google_drive_test.go b/internal/syncer/connector/google_drive_test.goindex 045188938f..5673caf66e 100644--- a/internal/syncer/connector/google_drive_test.go+++ b/internal/syncer/connector/google_drive_test.go@@ -92,6 +92,125 @@ func TestGoogleDriveConnectorOpenSyncUsesWindowFingerprintAndFetch(t *testing.T)}}+// TestGoogleDriveConnectorOpenSyncResumesWithinPage verifies Drive checkpoints resume inside a list page.+func TestGoogleDriveConnectorOpenSyncResumesWithinPage(t *testing.T) {+ connector, err := NewGoogleDriveConnector(map[string]any{+ "my_drive_emails": "[email protected]",+ "batch_size": 2,+ "credentials": map[string]any{+ "google_primary_admin": "[email protected]",+ "google_tokens": `{"client_id":"client","client_secret":"secret","refresh_token":"refresh"}`,+ },+ })+ if err != nil {+ t.Fatalf("NewGoogleDriveConnector failed: %v", err)+ }+ files := []googleDriveFile{+ googleDriveTestFile("file-1", "One.txt", "2026-01-01T00:00:00Z"),+ googleDriveTestFile("file-2", "Two.txt", "2026-01-02T00:00:00Z"),+ googleDriveTestFile("file-3", "Three.txt", "2026-01-03T00:00:00Z"),+ }+ var pageTokens []string+ connector.listFiles = func(ctx context.Context, userEmail string, request googleDriveListRequest) (googleDriveFilePage, error) {+ pageTokens = append(pageTokens, request.PageToken)+ return googleDriveFilePage{Files: files}, nil+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if len(first.Documents) != 2 || first.Documents[0].SourceID != "https://drive.google.com/file/d/file-1" || first.Documents[1].SourceID != "https://drive.google.com/file/d/file-2" {+ t.Fatalf("first documents = %+v", first.Documents)+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "https://drive.google.com/file/d/file-2" {+ t.Fatalf("first checkpoint = %+v, want file-2", first.Checkpoint)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://drive.google.com/file/d/file-3" {+ t.Fatalf("resume documents = %+v, want file-3", second.Documents)+ }+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "https://drive.google.com/file/d/file-3" {+ t.Fatalf("resume checkpoint = %+v, want file-3", second.Checkpoint)+ }+ if len(pageTokens) != 2 || pageTokens[1] != "" {+ t.Fatalf("page tokens = %v, want resume from first page", pageTokens)+ }+}++// TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged verifies page deletions resume from the current filename offset.+func TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {+ connector, err := NewGoogleDriveConnector(map[string]any{+ "my_drive_emails": "[email protected]",+ "batch_size": 2,+ "credentials": map[string]any{+ "google_primary_admin": "[email protected]",+ "google_tokens": `{"client_id":"client","client_secret":"secret","refresh_token":"refresh"}`,+ },+ })+ if err != nil {+ t.Fatalf("NewGoogleDriveConnector failed: %v", err)+ }+ files := []googleDriveFile{+ googleDriveTestFile("file-1", "One.txt", "2026-01-01T00:00:00Z"),+ googleDriveTestFile("file-2", "Two.txt", "2026-01-02T00:00:00Z"),+ googleDriveTestFile("file-3", "Three.txt", "2026-01-03T00:00:00Z"),+ }+ connector.listFiles = func(ctx context.Context, userEmail string, request googleDriveListRequest) (googleDriveFilePage, error) {+ return googleDriveFilePage{Files: files}, nil+ }+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "https://drive.google.com/file/d/file-2" {+ t.Fatalf("first checkpoint = %+v, want file-2", first.Checkpoint)+ }++ files = []googleDriveFile{+ googleDriveTestFile("file-1", "One.txt", "2026-01-01T00:00:00Z"),+ googleDriveTestFile("file-3", "Three.txt", "2026-01-03T00:00:00Z"),+ }+ resumeListCalls := 0+ connector.listFiles = func(ctx context.Context, userEmail string, request googleDriveListRequest) (googleDriveFilePage, error) {+ resumeListCalls+++ if request.PageToken != "" {+ t.Fatalf("resume used unexpected pageToken=%q", request.PageToken)+ }+ return googleDriveFilePage{Files: files}, nil+ }+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://drive.google.com/file/d/file-3" {+ t.Fatalf("resume documents = %+v, want file-3", second.Documents)+ }+ if resumeListCalls != 1 {+ t.Fatalf("resume list calls = %d, want one checkpoint page", resumeListCalls)+ }+}+// TestGoogleDriveSharedFolderScopesRecurse verifies shared folders walk child folders.func TestGoogleDriveSharedFolderScopesRecurse(t *testing.T) {connector, err := NewGoogleDriveConnector(map[string]any{@@ -233,3 +352,13 @@ func TestGoogleDriveFileQueryUsesIncrementalWindow(t *testing.T) {t.Fatalf("query = %q", query)}}++func googleDriveTestFile(id, name, modifiedTime string) googleDriveFile {+ return googleDriveFile{+ ID: id,+ Name: name,+ MimeType: "text/plain",+ ModifiedTime: modifiedTime,+ WebViewLink: "https://drive.google.com/file/d/" + id + "/view",+ }+}diff --git a/internal/syncer/connector/mock/mock.go b/internal/syncer/connector/mock/mock.goindex 7a1e3c8bad..efe26780eb 100644--- a/internal/syncer/connector/mock/mock.go+++ b/internal/syncer/connector/mock/mock.go@@ -32,6 +32,7 @@ type Connector struct {FetchBlobs map[string][]byteOnSyncBatch func(index int)OnPruneBatch func(index int)+ SyncRequests []syncerconnector.SyncRequest}// Validate returns the configured validation error.@@ -41,6 +42,7 @@ func (c *Connector) Validate(ctx context.Context) error {// OpenSync opens a mock sync session.func (c *Connector) OpenSync(ctx context.Context, request syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {+ c.SyncRequests = append(c.SyncRequests, request)return &SyncSession{connector: c}, nil}diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.goindex 4454b0f866..f064a40871 100644--- a/internal/syncer/connector/models.go+++ b/internal/syncer/connector/models.go@@ -37,19 +37,48 @@ type FetchReference struct {SizeHint int64}+// SyncCheckpoint is a connector-owned resume point.+type SyncCheckpoint struct {+ Cursor string `json:"cursor,omitempty"`+ UpdatedAt *time.Time `json:"updated_at,omitempty"`+ SourceID string `json:"source_id,omitempty"`+}++// SyncCheckpointState is the running checkpoint for one sync task.+type SyncCheckpointState struct {+ Version int `json:"version"`+ TaskID string `json:"task_id"`+ ConnectorID string `json:"connector_id"`+ KBID string `json:"kb_id"`+ WindowStart *time.Time `json:"window_start,omitempty"`+ WindowEnd time.Time `json:"window_end"`+ NextCommitSeq int64 `json:"next_commit_seq"`+ Checkpoint *SyncCheckpoint `json:"checkpoint,omitempty"`+ Added int64 `json:"added,omitempty"`+ Updated int64 `json:"updated,omitempty"`+ Skipped int64 `json:"skipped,omitempty"`+ ErrorCount int64 `json:"error_count,omitempty"`+ ErrorMsg string `json:"error_msg,omitempty"`+}+// SyncRequest describes one fixed sync window.type SyncRequest struct {- TaskID string- ConnectorID string- KBID string+ TaskID string+ ConnectorID string+ KBID string+ SourceType string+ Fingerprints map[string]string+FromBeginning boolWindowStart *time.TimeWindowEnd time.Time+ Resume *SyncCheckpoint}// SyncBatch contains one serially processed batch.type SyncBatch struct {- Documents []SourceDocument+ Documents []SourceDocument+ Checkpoint *SyncCheckpoint}// PruneRequest describes one complete prune snapshot request.diff --git a/internal/syncer/job_executor.go b/internal/syncer/job_executor.goindex e6f7c8f6fb..2b4c67c1ab 100644--- a/internal/syncer/job_executor.go+++ b/internal/syncer/job_executor.go@@ -21,6 +21,7 @@ import ("errors""fmt""ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector""sync""time")@@ -53,14 +54,16 @@ type syncJobFunc func(context.Context) (service.SyncStats, error)// syncJobResult the statstype syncJobResult struct {- stats service.SyncStats- err error+ stats service.SyncStats+ checkpoint *syncerconnector.SyncCheckpoint+ err error}type syncJob struct {- ctx context.Context- fn syncJobFunc- done chan syncJobResult+ ctx context.Context+ fn syncJobFunc+ checkpoint *syncerconnector.SyncCheckpoint+ done chan syncJobResult}// SyncJobQueue is one Coordinator-owned queue feeding the fair dispatcher.@@ -73,12 +76,16 @@ type SyncJobQueue struct {// Submit adds one BatchJob to this task's dispatcher queue.func (q *SyncJobQueue) Submit(ctx context.Context, fn syncJobFunc) (<-chan syncJobResult, error) {+ return q.submit(ctx, fn, nil)+}++func (q *SyncJobQueue) submit(ctx context.Context, fn syncJobFunc, checkpoint *syncerconnector.SyncCheckpoint) (<-chan syncJobResult, error) {if q == nil {return nil, fmt.Errorf("sync job queue is nil")}done := make(chan syncJobResult, 1) // done channel- job := &syncJob{ctx: ctx, fn: fn, done: done}+ job := &syncJob{ctx: ctx, fn: fn, checkpoint: checkpoint, done: done}select {case <-ctx.Done():return nil, ctx.Err()@@ -335,6 +342,6 @@ func (e *SyncJobExecutor) work() {continue}stats, err := job.fn(job.ctx) // run the job- job.done <- syncJobResult{stats: stats, err: err}+ job.done <- syncJobResult{stats: stats, checkpoint: job.checkpoint, err: err}}}diff --git a/internal/syncer/prune_runner.go b/internal/syncer/prune_runner.goindex 219d42e057..8da242c6d3 100644--- a/internal/syncer/prune_runner.go+++ b/internal/syncer/prune_runner.go@@ -36,9 +36,9 @@ func NewPruneRunner(taskService *service.SyncTaskService, pruneService *service.}// Run collects the full prune snapshot before deleting stale documents.-func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {+func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) (string, error) {if r.pruneService == nil {- return errors.New("prune service is not configured")+ return "", errors.New("prune service is not configured")}// Get the run sessionsession, err := connector.OpenPrune(ctx, syncerconnector.PruneRequest{@@ -47,21 +47,21 @@ func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskConteKBID: taskContext.Knowledgebase.ID,})if err != nil {- return err+ return "", err}defer session.Close()retain := map[string]struct{}{}for {if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {- return err+ return "", err}batch, nextErr := session.NextBatch(ctx)if errors.Is(nextErr, io.EOF) {break}if nextErr != nil {- return nextErr+ return "", nextErr}for _, doc := range batch.Documents {service.AddRetainDocumentID(retain, taskContext.Knowledgebase.ID, taskContext.Connector.ID, doc.SourceID)@@ -69,11 +69,11 @@ func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskConte}if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {- return err+ return "", err}removed, err := r.pruneService.DeleteStale(ctx, taskContext, retain)if err != nil {- return err+ return "", err}return r.taskService.CompletePrune(ctx, taskContext, removed)}diff --git a/internal/syncer/scheduler.go b/internal/syncer/scheduler.goindex f51da67ec5..fe17e2954c 100644--- a/internal/syncer/scheduler.go+++ b/internal/syncer/scheduler.go@@ -20,8 +20,12 @@ import ("context""errors""ragflow/internal/common"+ "ragflow/internal/entity""ragflow/internal/service"+ "sync""time"++ "go.uber.org/zap")// TaskEnvelope is the only payload sent through the task queue.@@ -31,34 +35,36 @@ type TaskEnvelope struct {stopHeartbeat func()}-// SyncTaskBroker publishes and pulls syncer task wake-up messages.+// SyncTaskBroker publishes and subscribes to syncer task wake-up messages.type SyncTaskBroker interface {InitSyncerStream() errorInitSyncerConsumer() errorPublishSyncerTask(taskID string) error- FetchSyncerTasks(batchSize int) ([]common.TaskHandle, error)+ PublishSyncerTaskWakeup(taskID string) error+ SubscribeSyncerTasks(ctx context.Context, handler func(common.TaskHandle)) error}// Scheduler discovers due work and enqueues task IDs for workers.type Scheduler struct {- pollInterval time.Duration- queue chan<- TaskEnvelope- taskService *service.SyncTaskService- broker SyncTaskBroker+ queue chan<- TaskEnvelope+ taskService *service.SyncTaskService+ broker SyncTaskBroker+ timerMu sync.Mutex+ timers map[string]*time.Timer}// NewScheduler creates a global scheduler for datasource sync tasks.-func NewScheduler(pollInterval time.Duration, queue chan<- TaskEnvelope, taskService *service.SyncTaskService) *Scheduler {- return &Scheduler{pollInterval: pollInterval, queue: queue, taskService: taskService}+func NewScheduler(queue chan<- TaskEnvelope, taskService *service.SyncTaskService) *Scheduler {+ return &Scheduler{queue: queue, taskService: taskService, timers: map[string]*time.Timer{}}}// NewNATSScheduler creates a JetStream-driven scheduler with DB reconciliation.-func NewNATSScheduler(pollInterval time.Duration, queue chan<- TaskEnvelope, taskService *service.SyncTaskService, broker SyncTaskBroker) *Scheduler {+func NewNATSScheduler(queue chan<- TaskEnvelope, taskService *service.SyncTaskService, broker SyncTaskBroker) *Scheduler {return &Scheduler{- pollInterval: pollInterval,- queue: queue,- taskService: taskService,- broker: broker,+ queue: queue,+ taskService: taskService,+ broker: broker,+ timers: map[string]*time.Timer{},}}@@ -77,127 +83,162 @@ func (s *Scheduler) runNATS(ctx context.Context) error {if err := s.broker.InitSyncerConsumer(); err != nil {return err}+ if err := s.broker.SubscribeSyncerTasks(ctx, func(handle common.TaskHandle) {+ if err := s.enqueueHandle(ctx, handle); err != nil && ctx.Err() == nil {+ common.Error("syncer scheduler enqueue failed", err)+ }+ }); err != nil {+ return err+ }// scan DB for first time runif err := s.publishStartupTasks(ctx); err != nil && ctx.Err() == nil {common.Error("syncer scheduler startup publish failed", err)}- ticker := time.NewTicker(s.pollInterval)- defer ticker.Stop()- for {- if err := ctx.Err(); err != nil {- return err- }-- select {- case <-ctx.Done():- return ctx.Err()- case <-ticker.C: // Run `publishDueTasks` periodically- if err := s.publishDueTasks(ctx); err != nil && ctx.Err() == nil {- common.Error("syncer scheduler due publish failed", err)- }- continue- default:- }-- // check `scheduler`'s task queue's slot- available := s.queueAvailable()- if available <= 0 {- if err := waitNATSFetchCapacity(ctx); err != nil {- return err- }- continue- }+ <-ctx.Done()+ s.stopTimers()+ return ctx.Err()+}- // pull `available` tasks from nats- handles, err := s.broker.FetchSyncerTasks(available)- if err != nil {- common.Error("syncer scheduler fetch failed", err)- if waitErr := waitNATSFetchCapacity(ctx); waitErr != nil {- return waitErr- }- } else if len(handles) == 0 {- if waitErr := waitNATSFetchCapacity(ctx); waitErr != nil {- return waitErr- }- } else if err = s.enqueueHandles(ctx, handles); err != nil { // put tasks to task queue- return err- }+func (s *Scheduler) enqueueHandle(ctx context.Context, handle common.TaskHandle) error {+ message := handle.GetMessage()+ if message.TaskID == "" {+ _ = handle.Ack()+ return nil}-}-func waitNATSFetchCapacity(ctx context.Context) error {+ stopHeartbeat := startHandleHeartbeat(ctx, handle)select {case <-ctx.Done():+ stopHeartbeat()+ _ = handle.Nack()return ctx.Err()- case <-time.After(100 * time.Millisecond):+ case s.queue <- TaskEnvelope{TaskID: message.TaskID, Handle: handle, stopHeartbeat: stopHeartbeat}:return nil}}-// enqueueHandles put task to `scheduler`'s task queue-func (s *Scheduler) enqueueHandles(ctx context.Context, handles []common.TaskHandle) error {- for index, handle := range handles {- message := handle.GetMessage()- if message.TaskID == "" {- _ = handle.Ack()- continue- }-- stopHeartbeat := startHandleHeartbeat(ctx, handle)- select {- case <-ctx.Done():- stopHeartbeat()- for _, pending := range handles[index:] {- _ = pending.Nack()- }- return ctx.Err()- case s.queue <- TaskEnvelope{TaskID: message.TaskID, Handle: handle, stopHeartbeat: stopHeartbeat}:- }- }- return nil-}-func (s *Scheduler) queueAvailable() int {return cap(s.queue) - len(s.queue)}-// publishStartupTasks scan DB for first time run+// publishStartupTasks scans DB once for startup reconciliation.func (s *Scheduler) publishStartupTasks(ctx context.Context) error {// TODO restores running tasks during syncer startup.if err := s.taskService.RecoverRunning(ctx); err != nil {return err}- tasks, err := s.taskService.ListStartupTasks(ctx)+ tasks, err := s.taskService.ListScheduledTasks(ctx)if err != nil {return err}- // publish task to nats- var publishErr errorfor _, task := range tasks {- if err = s.broker.PublishSyncerTask(task.ID); err != nil {- publishErr = errors.Join(publishErr, err)+ if err = s.ScheduleTask(ctx, task); err != nil {+ return err}}- return publishErr+ return nil}-// publishDueTasks publishes due scheduled DB tasks to nats-func (s *Scheduler) publishDueTasks(ctx context.Context) error {- now := time.Now()- tasks, err := s.taskService.ListDueTasks(ctx, now)- if err != nil {+// ScheduleTask publishes a due scheduled task or arms a one-shot timer.+func (s *Scheduler) ScheduleTask(ctx context.Context, task service.ScheduledSyncTask) error {+ delay, schedule := s.taskDelay(task, time.Now())+ if !schedule {+ return nil+ }+ return s.ScheduleTaskAfter(ctx, task.ID, delay)+}++// ScheduleTaskAfter publishes a task after delay.+func (s *Scheduler) ScheduleTaskAfter(ctx context.Context, taskID string, delay time.Duration) error {+ if s == nil || s.broker == nil || taskID == "" {+ return nil+ }+ if delay <= 0 {+ return s.publishTask(ctx, taskID)+ }+ s.timerMu.Lock()+ if existing := s.timers[taskID]; existing != nil {+ existing.Stop()+ }+ timer := time.AfterFunc(delay, func() {+ if err := s.publishTaskWakeup(ctx, taskID); err != nil && ctx.Err() == nil {+ common.Warn("syncer scheduler timer publish failed", zap.String("task_id", taskID), zap.Error(err))+ _ = s.ScheduleTaskAfter(ctx, taskID, 3*time.Second)+ return+ }+ s.timerMu.Lock()+ delete(s.timers, taskID)+ s.timerMu.Unlock()+ })+ s.timers[taskID] = timer+ s.timerMu.Unlock()+ return nil+}++func (s *Scheduler) publishTaskWakeup(ctx context.Context, taskID string) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ if err := s.broker.PublishSyncerTaskWakeup(taskID); err != nil {+ common.Warn("syncer task wakeup publish failed", zap.String("task_id", taskID), zap.Error(err))return err}+ return nil+}- var publishErr error- for _, task := range tasks {- if err = s.broker.PublishSyncerTask(task.ID); err != nil {- publishErr = errors.Join(publishErr, err)+func (s *Scheduler) publishTask(ctx context.Context, taskID string) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ if err := s.broker.PublishSyncerTask(taskID); err != nil {+ common.Warn("syncer task publish failed", zap.String("task_id", taskID), zap.Error(err))+ return err+ }+ return nil+}++func (s *Scheduler) stopTimers() {+ s.timerMu.Lock()+ defer s.timerMu.Unlock()+ for taskID, timer := range s.timers {+ timer.Stop()+ delete(s.timers, taskID)+ }+}++// taskDelay reports the delay before publication and whether the task must be scheduled at all.+func (s *Scheduler) taskDelay(task service.ScheduledSyncTask, now time.Time) (time.Duration, bool) {+ freq := int64(0)+ switch task.TaskType {+ case service.TaskTypeSync:+ freq = task.ConnectorRefreshFreq+ case service.TaskTypePrune:+ if !syncerConfigBool(task.ConnectorConfig, "sync_deleted_files") {+ return 0, false}+ freq = task.ConnectorPruneFreq+ }+ if freq <= 0 || task.UpdateDate == nil {+ return 0, true+ }+ return task.UpdateDate.Add(time.Duration(freq) * time.Minute).Sub(now), true+}++func syncerConfigBool(config entity.JSONMap, key string) bool {+ value, ok := config[key]+ if !ok {+ return false+ }+ switch typed := value.(type) {+ case bool:+ return typed+ case string:+ return typed == "1" || typed == "true" || typed == "TRUE"+ default:+ return false}- return publishErr}diff --git a/internal/syncer/sync_runner.go b/internal/syncer/sync_runner.goindex e2b7696765..279d68fcf1 100644--- a/internal/syncer/sync_runner.go+++ b/internal/syncer/sync_runner.go@@ -22,9 +22,12 @@ import ("fmt""io""math/rand"+ "ragflow/internal/common""ragflow/internal/service"syncerconnector "ragflow/internal/syncer/connector""time"++ "go.uber.org/zap")var errSyncTaskCanceled = errors.New("sync task canceled")@@ -38,49 +41,61 @@ type SyncRunner struct {sink service.DocumentSinkidResolver *service.DocumentIDResolverqueue *SyncJobQueue+ checkpoints SyncCheckpointStore}// NewSyncRunner creates a SYNC runner.-func NewSyncRunner(config TaskCoordinatorConfig, taskService *service.SyncTaskService, sink service.DocumentSink, idResolver *service.DocumentIDResolver, queue *SyncJobQueue) *SyncRunner {- return &SyncRunner{config: config, taskService: taskService, sink: sink, idResolver: idResolver, queue: queue}+func NewSyncRunner(config TaskCoordinatorConfig, taskService *service.SyncTaskService, sink service.DocumentSink, idResolver *service.DocumentIDResolver, queue *SyncJobQueue, checkpoints SyncCheckpointStore) *SyncRunner {+ if checkpoints == nil {+ checkpoints = newMemorySyncCheckpointStore()+ }+ return &SyncRunner{config: config, taskService: taskService, sink: sink, idResolver: idResolver, queue: queue, checkpoints: checkpoints}}// Run executes all sync batches and commits the final waterline.-func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {+func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) (string, error) {// sink is nil means this syncer task cannot write it to document, it will fail anywayif r.sink == nil {- return errors.New("document sink is not configured")+ return "", errors.New("document sink is not configured")}- windowEnd := time.Now().UTC()-var windowStart *time.Time // = nil if it is `Full synchronisation`if !service.IsFromBeginning(taskContext.Task.FromBeginning) {windowStart = taskContext.Task.PollRangeStart}+ checkpointState, err := r.prepareCheckpoint(ctx, taskContext, windowStart, time.Now().UTC())+ if err != nil {+ return "", err+ }+ sourceType := service.SourceType(taskContext.Connector.Source, taskContext.Connector.ID)+ fingerprints, err := r.idResolver.ListFingerprintsBySourceType(ctx, taskContext.Knowledgebase.ID, sourceType)+ if err != nil {+ return "", err+ }session, err := connector.OpenSync(ctx, syncerconnector.SyncRequest{TaskID: taskContext.Task.ID,ConnectorID: taskContext.Connector.ID,KBID: taskContext.Knowledgebase.ID,- FromBeginning: windowStart == nil,- WindowStart: windowStart,- WindowEnd: windowEnd,+ SourceType: sourceType,+ Fingerprints: fingerprints,+ FromBeginning: checkpointState.WindowStart == nil,+ WindowStart: checkpointState.WindowStart,+ WindowEnd: checkpointState.WindowEnd,+ Resume: checkpointState.Checkpoint,})if err != nil {- return err+ return "", err}defer session.Close()// prepare sourceType, waterline, stats, resultChan- sourceType := service.SourceType(taskContext.Connector.Source, taskContext.Connector.ID)- candidateEnd := windowStart // the waterLine that will write to DB- stats := service.SyncStats{} // count `add`, `updated`, `skipped`+ stats := statsFromCheckpointState(checkpointState) // count `add`, `updated`, `skipped`resultChans := make([]<-chan syncJobResult, 0)for {// check if task has been canceledif err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {- return err+ return "", err}// get a batch of files@@ -89,24 +104,40 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContexbreak}if nextErr != nil {- return nextErr- }-- for _, doc := range batch.Documents {- if candidateEnd == nil || doc.UpdatedAt.After(*candidateEnd) {- updatedAt := doc.UpdatedAt- candidateEnd = &updatedAt // use `max_update` to push `waterLine`+ if err = r.collectResults(ctx, taskContext.Task.ID, resultChans, &checkpointState, &stats); err != nil {+ return "", err}+ return "", nextErr}+// a batch, a syncJobresultChan, err := r.submitBatch(ctx, taskContext, sourceType, session, batch)if err != nil {- return err+ return "", err}resultChans = append(resultChans, resultChan)}- // run sync Job+ if err = r.collectResults(ctx, taskContext.Task.ID, resultChans, &checkpointState, &stats); err != nil {+ return "", err+ }++ if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {+ return "", err+ }+ nextTaskID, err := r.taskService.CompleteSync(ctx, taskContext, checkpointState.WindowEnd, stats)+ if err != nil {+ return "", err+ }+ if err = r.checkpoints.DeleteSyncCheckpoint(context.WithoutCancel(ctx), taskContext.Task.ID); err != nil {+ common.Warn("delete sync checkpoint failed", zap.Error(err))+ } else {+ common.Info("delete sync checkpoint completed", zap.String("task_id", taskContext.Task.ID))+ }+ return nextTaskID, nil+}++func (r *SyncRunner) collectResults(ctx context.Context, taskID string, resultChans []<-chan syncJobResult, checkpointState *syncerconnector.SyncCheckpointState, stats *service.SyncStats) error {var firstErr errorfor _, resultChan := range resultChans {var jobResult syncJobResult@@ -119,31 +150,97 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContexif jobResult.err != nil && firstErr == nil {firstErr = jobResult.err}+ if firstErr == nil && jobResult.checkpoint != nil {+ checkpointState.Checkpoint = cloneSyncCheckpoint(jobResult.checkpoint)+ checkpointState.NextCommitSeq+++ applyStatsToCheckpointState(checkpointState, *stats)+ if err := r.checkpoints.SaveSyncCheckpoint(ctx, taskID, *checkpointState); err != nil {+ return err+ }+ }}if firstErr != nil {return firstErr}+ return nil+}- if err := r.checkCanceled(ctx, taskContext.Task.ID); err != nil {- return err+func (r *SyncRunner) prepareCheckpoint(ctx context.Context, taskContext service.SyncTaskContext, windowStart *time.Time, windowEnd time.Time) (syncerconnector.SyncCheckpointState, error) {+ state, err := r.checkpoints.LoadSyncCheckpoint(ctx, taskContext.Task.ID)+ if err != nil {+ return syncerconnector.SyncCheckpointState{}, err}- if candidateEnd == nil {- candidateEnd = &windowEnd+ if state != nil && !state.WindowEnd.IsZero() && state.TaskID == taskContext.Task.ID && state.ConnectorID == taskContext.Connector.ID && state.KBID == taskContext.Knowledgebase.ID {+ if state.Version == 0 {+ state.Version = 1+ }+ if state.NextCommitSeq == 0 {+ state.NextCommitSeq = 1+ }+ return *state, nil+ }+ // if checkpoint is not exist+ initial := syncerconnector.SyncCheckpointState{+ Version: 1,+ TaskID: taskContext.Task.ID,+ ConnectorID: taskContext.Connector.ID,+ KBID: taskContext.Knowledgebase.ID,+ WindowStart: windowStart,+ WindowEnd: windowEnd,+ NextCommitSeq: 1,}- return r.taskService.CompleteSync(ctx, taskContext, *candidateEnd, stats)+ if err = r.checkpoints.SaveSyncCheckpoint(ctx, taskContext.Task.ID, initial); err != nil {+ return syncerconnector.SyncCheckpointState{}, err+ }+ return initial, nil+}++func statsFromCheckpointState(state syncerconnector.SyncCheckpointState) service.SyncStats {+ return service.SyncStats{+ Added: state.Added,+ Updated: state.Updated,+ Skipped: state.Skipped,+ ErrorCount: state.ErrorCount,+ ErrorMsg: state.ErrorMsg,+ }+}++// applyStatsToCheckpointState store checkpoint's state data+func applyStatsToCheckpointState(state *syncerconnector.SyncCheckpointState, stats service.SyncStats) {+ state.Added = stats.Added+ state.Updated = stats.Updated+ state.Skipped = stats.Skipped+ state.ErrorCount = stats.ErrorCount+ state.ErrorMsg = stats.ErrorMsg}// submitBatch submits one source batch as one BatchJob.func (r *SyncRunner) submitBatch(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, batch syncerconnector.SyncBatch) (<-chan syncJobResult, error) {- resultChan, err := r.queue.Submit(ctx, func(jobCtx context.Context) (service.SyncStats, error) {+ resultChan, err := r.queue.submit(ctx, func(jobCtx context.Context) (service.SyncStats, error) {return r.processDocuments(jobCtx, taskContext, sourceType, session, batch.Documents)- })+ }, batch.Checkpoint)if err != nil {return nil, err}return resultChan, nil}+// cloneSyncCheckpoint safe clone+func cloneSyncCheckpoint(checkpoint *syncerconnector.SyncCheckpoint) *syncerconnector.SyncCheckpoint {+ if checkpoint == nil {+ return nil+ }+ clone := &syncerconnector.SyncCheckpoint{+ Cursor: checkpoint.Cursor,+ SourceID: checkpoint.SourceID,+ }+ if checkpoint.UpdatedAt != nil {+ updatedAt := *checkpoint.UpdatedAt+ clone.UpdatedAt = &updatedAt+ }+ return clone+}+// processDocumentsfunc (r *SyncRunner) processDocuments(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, documents []syncerconnector.SourceDocument) (service.SyncStats, error) {stats := service.SyncStats{}diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.goindex d41c7735dd..51ae09ce07 100644--- a/internal/syncer/syncer.go+++ b/internal/syncer/syncer.go@@ -49,11 +49,10 @@ type Syncer struct {}// NewSyncer creates a server-compatible syncer with default dependencies.-func NewSyncer(taskWorkerCount int, pollInterval time.Duration) *Syncer {+func NewSyncer(taskWorkerCount int) *Syncer {// init the configconfig := DefaultConfig()config.TaskWorkerCount = taskWorkerCount- config.PollInterval = pollIntervaltaskDAO := dao.NewSyncTaskDAO(nil)registry := syncerconnector.NewRegistry()@@ -78,15 +77,20 @@ func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, si})taskService := service.NewSyncTaskService(taskDAO)idResolver := service.NewDocumentIDResolver(service.NewGormDocumentStore())+ checkpoints := SyncCheckpointStore(newMemorySyncCheckpointStore())+ messageQueue := engine.GetMessageQueueEngine()+ if store, ok := messageQueue.(SyncCheckpointStore); ok {+ checkpoints = store+ }coordinator := NewTaskCoordinator(TaskCoordinatorConfig{ItemRetryCount: config.ItemRetryCount,ItemRetryBaseDelay: config.ItemRetryBaseDelay,- }, taskService, registry, sink, pruneService, idResolver, executor)+ }, taskService, registry, sink, pruneService, idResolver, executor, checkpoints)- scheduler := NewScheduler(config.PollInterval, queue, taskService)- if broker, ok := engine.GetMessageQueueEngine().(SyncTaskBroker); ok {- scheduler = NewNATSScheduler(config.PollInterval, queue, taskService, broker)+ scheduler := NewScheduler(queue, taskService)+ if broker, ok := messageQueue.(SyncTaskBroker); ok {+ scheduler = NewNATSScheduler(queue, taskService, broker)}return &Syncer{@@ -94,7 +98,7 @@ func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, siconfig: config,queue: queue,scheduler: scheduler,- worker: NewTaskWorker(queue, taskService, coordinator, locker),+ worker: NewTaskWorker(queue, taskService, coordinator, locker).WithScheduler(scheduler),executor: executor,ShutdownCh: make(chan struct{}),}@@ -108,11 +112,6 @@ func (s *Syncer) ID() string {return s.id}-// Start launches the scheduler and task workers with a background context.-func (s *Syncer) Start() error {- return s.StartContext(context.Background())-}-// StartContext launches the scheduler and task workers.func (s *Syncer) StartContext(ctx context.Context) error {if s == nil {diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.goindex a6bec5a3d5..59136280e4 100644--- a/internal/syncer/syncer_test.go+++ b/internal/syncer/syncer_test.go@@ -101,6 +101,14 @@ func (s fakeStore) ListIDs(ctx context.Context, kbID, sourceType string) (map[streturn s.ids, nil}+// ListFingerprintsBySourceType returns configured fingerprints.+func (s fakeStore) ListFingerprintsBySourceType(ctx context.Context, kbID, sourceType string) (map[string]string, error) {+ if s.fingerprints == nil {+ return map[string]string{}, nil+ }+ return s.fingerprints, nil+}+// GetFingerprintsByIDs returns configured fingerprints for requested IDs.func (s fakeStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {result := make(map[string]string, len(ids))@@ -156,26 +164,30 @@ func (h *fakeTaskHandle) counts() (int, int) {}type fakeSyncTaskBroker struct {+ mu sync.Mutexpublished []string- fetched []common.TaskHandle+ handler func(common.TaskHandle)}func (b *fakeSyncTaskBroker) InitSyncerStream() error { return nil }func (b *fakeSyncTaskBroker) InitSyncerConsumer() error { return nil }func (b *fakeSyncTaskBroker) PublishSyncerTask(taskID string) error {+ b.mu.Lock()+ defer b.mu.Unlock()b.published = append(b.published, taskID)return nil}-func (b *fakeSyncTaskBroker) FetchSyncerTasks(batchSize int) ([]common.TaskHandle, error) {- if len(b.fetched) == 0 {- return nil, nil- }- if batchSize > len(b.fetched) {- batchSize = len(b.fetched)- }- out := b.fetched[:batchSize]- b.fetched = b.fetched[batchSize:]- return out, nil+func (b *fakeSyncTaskBroker) PublishSyncerTaskWakeup(taskID string) error {+ return b.PublishSyncerTask(taskID)+}+func (b *fakeSyncTaskBroker) SubscribeSyncerTasks(ctx context.Context, handler func(common.TaskHandle)) error {+ b.handler = handler+ return nil+}+func (b *fakeSyncTaskBroker) publishedIDs() []string {+ b.mu.Lock()+ defer b.mu.Unlock()+ return append([]string(nil), b.published...)}// DeleteDocument records one delete.@@ -308,8 +320,12 @@ func newTestRegistry(connectors map[string]*connectormock.Connector) *syncerconn// newCoordinator creates a test coordinator.func newCoordinator(taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore) *TaskCoordinator {+ return newCoordinatorWithCheckpoints(taskService, registry, sink, pruneService, store, newMemorySyncCheckpointStore())+}++func newCoordinatorWithCheckpoints(taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore, checkpoints SyncCheckpointStore) *TaskCoordinator {executor := NewSyncJobExecutor(SyncJobExecutorConfig{WorkerCount: 16})- return NewTaskCoordinator(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), executor)+ return NewTaskCoordinator(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), executor, checkpoints)}// TestClaimBlocksSameConnectorKBRunningTasks verifies DB-backed task mutual exclusion.@@ -354,7 +370,7 @@ func TestClaimAllowsSameConnectorDifferentKB(t *testing.T) {// TestSchedulerRequiresBroker verifies JetStream is mandatory for the scheduler.func TestSchedulerRequiresBroker(t *testing.T) {- scheduler := NewScheduler(time.Hour, make(chan TaskEnvelope, 1), service.NewSyncTaskService(dao.NewSyncTaskDAO(nil)))+ scheduler := NewScheduler(make(chan TaskEnvelope, 1), service.NewSyncTaskService(dao.NewSyncTaskDAO(nil)))err := scheduler.Run(t.Context())if err == nil || !strings.Contains(err.Error(), "NATS broker") {t.Fatalf("Run error = %v, want missing broker", err)@@ -367,7 +383,7 @@ func TestNATSSchedulerPublishesDueTasksWithoutClaiming(t *testing.T) {insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))broker := &fakeSyncTaskBroker{}- scheduler := NewNATSScheduler(time.Hour, make(chan TaskEnvelope, 1), taskService, broker)+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)if err := scheduler.publishStartupTasks(t.Context()); err != nil {t.Fatalf("publish startup tasks: %v", err)}@@ -383,8 +399,8 @@ func TestNATSSchedulerPublishesDueTasksWithoutClaiming(t *testing.T) {}}-// TestNATSSchedulerStartupPublishesScheduledTaskWithRefreshFreq verifies startup does not wait for the next refresh window.-func TestNATSSchedulerStartupPublishesScheduledTaskWithRefreshFreq(t *testing.T) {+// TestNATSSchedulerStartupDelaysFreshScheduledTask verifies startup keeps refresh windows without periodic DB scans.+func TestNATSSchedulerStartupDelaysFreshScheduledTask(t *testing.T) {db := setupSyncerDB(t)insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)now := time.Now()@@ -401,12 +417,18 @@ func TestNATSSchedulerStartupPublishesScheduledTaskWithRefreshFreq(t *testing.T)taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))broker := &fakeSyncTaskBroker{}- scheduler := NewNATSScheduler(time.Hour, make(chan TaskEnvelope, 1), taskService, broker)+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)if err := scheduler.publishStartupTasks(t.Context()); err != nil {t.Fatalf("publish startup tasks: %v", err)}- if len(broker.published) != 1 || broker.published[0] != "task-1" {- t.Fatalf("published = %v", broker.published)+ if got := broker.publishedIDs(); len(got) != 0 {+ t.Fatalf("published = %v, want delayed task", got)+ }+ scheduler.timerMu.Lock()+ _, scheduled := scheduler.timers["task-1"]+ scheduler.timerMu.Unlock()+ if !scheduled {+ t.Fatalf("task timer was not scheduled")}}@@ -422,13 +444,13 @@ func TestNATSSchedulerRecoversRunningTasksOnStartup(t *testing.T) {}taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))broker := &fakeSyncTaskBroker{}- scheduler := NewNATSScheduler(time.Hour, make(chan TaskEnvelope, 1), taskService, broker)+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)if err := scheduler.publishStartupTasks(t.Context()); err != nil {t.Fatalf("publish startup tasks: %v", err)}- if len(broker.published) != 1 || broker.published[0] != "task-1" {- t.Fatalf("published = %v", broker.published)+ if got := broker.publishedIDs(); len(got) != 1 || got[0] != "task-1" {+ t.Fatalf("published = %v", got)}var task entity.SyncLogsif err := db.First(&task, "id = ?", "task-1").Error; err != nil {@@ -446,8 +468,8 @@ func TestNATSSchedulerRecoversRunningTasksOnStartup(t *testing.T) {}}-// TestNATSSchedulerPublishesOnlyDueTasks verifies periodic NATS publishing respects refresh windows.-func TestNATSSchedulerPublishesOnlyDueTasks(t *testing.T) {+// TestNATSSchedulerStartupPublishesDueTasks verifies startup scan immediately publishes due work.+func TestNATSSchedulerStartupPublishesDueTasks(t *testing.T) {db := setupSyncerDB(t)insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)now := time.Now()@@ -463,13 +485,13 @@ func TestNATSSchedulerPublishesOnlyDueTasks(t *testing.T) {}taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))broker := &fakeSyncTaskBroker{}- scheduler := NewNATSScheduler(10*time.Millisecond, make(chan TaskEnvelope, 1), taskService, broker)+ scheduler := NewNATSScheduler(make(chan TaskEnvelope, 1), taskService, broker)- if err := scheduler.publishDueTasks(t.Context()); err != nil {- t.Fatalf("publish fresh due tasks: %v", err)+ if err := scheduler.publishStartupTasks(t.Context()); err != nil {+ t.Fatalf("publish fresh startup tasks: %v", err)}- if len(broker.published) != 0 {- t.Fatalf("fresh task published = %v, want none", broker.published)+ if got := broker.publishedIDs(); len(got) != 0 {+ t.Fatalf("fresh task published = %v, want none", got)}dueAt := now.Add(-10 * time.Minute)@@ -480,26 +502,28 @@ func TestNATSSchedulerPublishesOnlyDueTasks(t *testing.T) {t.Fatalf("set due task update time: %v", err)}- if err := scheduler.publishDueTasks(t.Context()); err != nil {- t.Fatalf("publish due tasks: %v", err)+ if err := scheduler.publishStartupTasks(t.Context()); err != nil {+ t.Fatalf("publish due startup tasks: %v", err)}- if len(broker.published) != 1 || broker.published[0] != "task-1" {- t.Fatalf("published = %v, want due task", broker.published)+ if got := broker.publishedIDs(); len(got) != 1 || got[0] != "task-1" {+ t.Fatalf("published = %v, want due task", got)}}-// TestNATSSchedulerBuffersFetchedTasks verifies NATS mode stores excess tasks in the local queue.-func TestNATSSchedulerBuffersFetchedTasks(t *testing.T) {+// TestNATSSchedulerBuffersPushedTasks verifies enqueueHandle buffers pushed NATS handles in the local queue.+func TestNATSSchedulerBuffersPushedTasks(t *testing.T) {taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(nil))queue := make(chan TaskEnvelope, 2)- scheduler := NewNATSScheduler(time.Hour, queue, taskService, &fakeSyncTaskBroker{})+ scheduler := NewNATSScheduler(queue, taskService, &fakeSyncTaskBroker{})handles := []common.TaskHandle{&fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-1", TaskType: common.TaskTypeSyncer}},&fakeTaskHandle{msg: common.TaskMessage{TaskID: "task-2", TaskType: common.TaskTypeSyncer}},}- if err := scheduler.enqueueHandles(t.Context(), handles); err != nil {- t.Fatalf("enqueue handles: %v", err)+ for _, handle := range handles {+ if err := scheduler.enqueueHandle(t.Context(), handle); err != nil {+ t.Fatalf("enqueue handle: %v", err)+ }}if got := scheduler.queueAvailable(); got != 0 {t.Fatalf("queue capacity while buffered = %d, want 0", got)@@ -717,7 +741,7 @@ func TestSyncRunnerSubmitsBatchesBeforeWaiting(t *testing.T) {sink := &fakeSink{delay: 80 * time.Millisecond}coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{})taskContext, _ := taskService.GetContext(t.Context(), "task-1")- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {t.Fatalf("execute: %v", err)}if secondBatchAt.Sub(start) >= 70*time.Millisecond {@@ -740,7 +764,7 @@ func TestSyncRunnerProcessesBatchJobsInParallel(t *testing.T) {sink := &fakeSink{delay: 80 * time.Millisecond}coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{})taskContext, _ := taskService.GetContext(t.Context(), "task-1")- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {t.Fatalf("execute: %v", err)}sink.mu.Lock()@@ -778,7 +802,7 @@ func TestFingerprintSkipsUnchangedDocument(t *testing.T) {sink := &fakeSink{}coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, store)taskContext, _ := taskService.GetContext(t.Context(), "task-1")- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {t.Fatalf("execute: %v", err)}if sink.callCount() != 0 {@@ -799,7 +823,7 @@ func TestAutoParseFlagFlowsToSink(t *testing.T) {sink := &fakeSink{}coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{})taskContext, _ := taskService.GetContext(t.Context(), "task-1")- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {t.Fatalf("execute: %v", err)}if sink.autoParseByDoc["source-1"] {@@ -818,7 +842,7 @@ func TestCompleteSyncSchedulesNextRun(t *testing.T) {connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("x"), UpdatedAt: now}}}}}coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{})taskContext, _ := taskService.GetContext(t.Context(), "task-1")- if err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {+ if _, err := coordinator.Execute(t.Context(), taskContext, testLockLease()); err != nil {t.Fatalf("execute: %v", err)}@@ -909,12 +933,13 @@ func TestSyncRunnerResultWaitHonorsCancel(t *testing.T) {}connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}queue := &SyncJobQueue{taskID: "task-1", jobs: make(chan *syncJob, 1)}- runner := NewSyncRunner(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, &fakeSink{}, service.NewDocumentIDResolver(fakeStore{}), queue)+ runner := NewSyncRunner(TaskCoordinatorConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, &fakeSink{}, service.NewDocumentIDResolver(fakeStore{}), queue, newMemorySyncCheckpointStore())ctx, cancel := context.WithCancel(context.Background())result := make(chan error, 1)go func() {- result <- runner.Run(ctx, taskContext, connector)+ _, err := runner.Run(ctx, taskContext, connector)+ result <- err}()select {@@ -957,6 +982,272 @@ func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {}}+// TestTransientFailureReschedulesTask verifies retryable source failures keep the same task scheduled.+func TestTransientFailureReschedulesTask(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ connector := &connectormock.Connector{SyncErrAt: 1}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusSchedule {+ t.Fatalf("status = %s, want schedule", task.Status)+ }+ if task.ErrorCount != 1 {+ t.Fatalf("error_count = %d, want 1", task.ErrorCount)+ }+ if !strings.Contains(task.ErrorMsg, "unexpected EOF") {+ t.Fatalf("error_msg = %q, want unexpected EOF", task.ErrorMsg)+ }+}++// TestTransientFailureFailsAfterThreeRetries verifies retryable errors become terminal after the retry budget.+func TestTransientFailureFailsAfterThreeRetries(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "error_count": int64(2)}).Error; err != nil {+ t.Fatalf("mark running: %v", err)+ }+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ connector := &connectormock.Connector{SyncErrAt: 1}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusFail {+ t.Fatalf("status = %s, want fail", task.Status)+ }+ if task.ErrorCount != 3 {+ t.Fatalf("error_count = %d, want 3", task.ErrorCount)+ }+ if !strings.Contains(task.ErrorMsg, "failed after 3 transient retries") || !strings.Contains(task.ErrorMsg, "unexpected EOF") {+ t.Fatalf("error_msg = %q", task.ErrorMsg)+ }+}++// TestTransientFetchFailureSavesCompletedBatchCheckpoint verifies source failures do not discard completed batch progress.+func TestTransientFetchFailureSavesCompletedBatchCheckpoint(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ checkpoints := newMemorySyncCheckpointStore()+ firstTime := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)+ connector := &connectormock.Connector{+ SyncErrAt: 2,+ SyncBatches: []syncerconnector.SyncBatch{{+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("a"), UpdatedAt: firstTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &firstTime, SourceID: "source-1"},+ }},+ }+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}, checkpoints), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("load checkpoint: %v", err)+ }+ if state == nil || state.Checkpoint == nil {+ t.Fatalf("checkpoint was not saved")+ }+ if state.Checkpoint.Cursor != "cursor-1" || state.Checkpoint.SourceID != "source-1" {+ t.Fatalf("checkpoint = %+v, want cursor-1/source-1", state.Checkpoint)+ }+ if state.Added != 1 {+ t.Fatalf("checkpoint added = %d, want 1", state.Added)+ }++ var task entity.SyncLogs+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusSchedule {+ t.Fatalf("status = %s, want schedule", task.Status)+ }+}++// TestSyncCheckpointDeletedAfterSuccess verifies completed tasks remove their checkpoint.+func TestSyncCheckpointDeletedAfterSuccess(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ checkpoints := newMemorySyncCheckpointStore()++ firstTime := time.Date(2026, 8, 11, 1, 0, 0, 0, time.UTC)+ secondTime := time.Date(2026, 8, 11, 1, 1, 0, 0, time.UTC)+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{+ {+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("a"), UpdatedAt: firstTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &firstTime, SourceID: "source-1"},+ },+ {+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-2", Blob: []byte("b"), UpdatedAt: secondTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-2", UpdatedAt: &secondTime, SourceID: "source-2"},+ },+ }}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}, checkpoints), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("load checkpoint: %v", err)+ }+ if state != nil {+ t.Fatalf("checkpoint = %+v, want deleted", state)+ }+}++// TestSyncCheckpointStopsBeforeFailedBatch verifies failed batches do not advance the resume point.+func TestSyncCheckpointStopsBeforeFailedBatch(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ checkpoints := newMemorySyncCheckpointStore()++ firstTime := time.Date(2026, 8, 11, 1, 0, 0, 0, time.UTC)+ secondTime := time.Date(2026, 8, 11, 1, 1, 0, 0, time.UTC)+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{+ {+ Documents: []syncerconnector.SourceDocument{{SourceID: "good", Blob: []byte("a"), UpdatedAt: firstTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", UpdatedAt: &firstTime, SourceID: "good"},+ },+ {+ Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("b"), UpdatedAt: secondTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-2", UpdatedAt: &secondTime, SourceID: "bad"},+ },+ }}+ sink := &fakeSink{errBySourceID: map[string]error{"bad": errors.New("boom")}}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, checkpoints), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})++ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("load checkpoint: %v", err)+ }+ if state == nil || state.Checkpoint == nil {+ t.Fatalf("checkpoint was not saved")+ }+ if state.Checkpoint.Cursor != "cursor-1" {+ t.Fatalf("checkpoint cursor = %q, want cursor-1", state.Checkpoint.Cursor)+ }+ if state.NextCommitSeq != 2 {+ t.Fatalf("next commit seq = %d, want 2", state.NextCommitSeq)+ }+ if state.Added != 1 || state.Updated != 0 {+ t.Fatalf("checkpoint stats added/updated = %d/%d, want 1/0", state.Added, state.Updated)+ }++ if err = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error; err != nil {+ t.Fatalf("mark running: %v", err)+ }+ thirdTime := secondTime.Add(time.Minute)+ resumeConnector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{+ Documents: []syncerconnector.SourceDocument{{SourceID: "second-good", Blob: []byte("c"), UpdatedAt: thirdTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-3", UpdatedAt: &thirdTime, SourceID: "second-good"},+ }}}+ resumeWorker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinatorWithCheckpoints(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": resumeConnector}), &fakeSink{}, nil, fakeStore{}, checkpoints), NewConnectorLock())+ resumeWorker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})+ if len(resumeConnector.SyncRequests) != 1 {+ t.Fatalf("sync requests = %d, want 1", len(resumeConnector.SyncRequests))+ }+ request := resumeConnector.SyncRequests[0]+ if request.Resume == nil || request.Resume.Cursor != "cursor-1" || request.Resume.SourceID != "good" {+ t.Fatalf("resume = %+v, want cursor-1/good", request.Resume)+ }+ if !request.WindowEnd.Equal(state.WindowEnd) {+ t.Fatalf("window end = %s, want %s", request.WindowEnd, state.WindowEnd)+ }+ var task entity.SyncLogs+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load completed task: %v", err)+ }+ if task.Status != dao.SyncStatusDone {+ t.Fatalf("status = %s, want done", task.Status)+ }+ if task.TotalDocsIndexed != 2 {+ t.Fatalf("total docs indexed = %d, want 2", task.TotalDocsIndexed)+ }+}++// TestSyncRunnerClampsWaterlineToWindowEnd verifies future source timestamps do not poison the next window.+func TestSyncRunnerClampsWaterlineToWindowEnd(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ start := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)+ if err := db.Model(&entity.SyncLogs{}).+ Where("id = ?", "task-1").+ Updates(map[string]any{"status": dao.SyncStatusRunning, "poll_range_start": &start}).Error; err != nil {+ t.Fatalf("mark running: %v", err)+ }++ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ future := time.Date(2036, 6, 28, 9, 44, 0, 0, time.UTC)+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{+ Documents: []syncerconnector.SourceDocument{{SourceID: "future", Blob: []byte("future"), UpdatedAt: future}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-future", UpdatedAt: &future, SourceID: "future"},+ }}}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())+ before := time.Now().UTC()+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})+ after := time.Now().UTC().Add(time.Second)++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.PollRangeEnd == nil {+ t.Fatalf("poll_range_end is nil")+ }+ if task.PollRangeEnd.Equal(future) || task.PollRangeEnd.After(after) || task.PollRangeEnd.Before(before.Add(-time.Second)) {+ t.Fatalf("poll_range_end = %s, want clamped to run window around %s - %s", task.PollRangeEnd, before, after)+ }+}++// TestFullSyncWaterlineUsesWindowEnd verifies successful full sync does not keep the waterline at the latest source timestamp.+func TestFullSyncWaterlineUsesWindowEnd(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ fromBeginning := "1"+ if err := db.Model(&entity.SyncLogs{}).+ Where("id = ?", "task-1").+ Updates(map[string]any{"status": dao.SyncStatusRunning, "from_beginning": &fromBeginning}).Error; err != nil {+ t.Fatalf("mark running: %v", err)+ }++ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ oldSourceTime := time.Date(2025, 5, 17, 12, 39, 23, 0, time.UTC)+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{+ Documents: []syncerconnector.SourceDocument{{SourceID: "old", Blob: []byte("old"), UpdatedAt: oldSourceTime}},+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-old", UpdatedAt: &oldSourceTime, SourceID: "old"},+ }}}+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, nil, fakeStore{}), NewConnectorLock())+ before := time.Now().UTC()+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})+ after := time.Now().UTC().Add(time.Second)++ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.PollRangeEnd == nil {+ t.Fatalf("poll_range_end is nil")+ }+ if task.PollRangeEnd.Equal(oldSourceTime) || task.PollRangeEnd.Before(before.Add(-time.Second)) || task.PollRangeEnd.After(after) {+ t.Fatalf("poll_range_end = %s, want run window around %s - %s", task.PollRangeEnd, before, after)+ }+}+// TestRecoverStaleRunningTasks verifies timeout recovery.func TestRecoverStaleRunningTasks(t *testing.T) {db := setupSyncerDB(t)diff --git a/internal/syncer/task_coordinator.go b/internal/syncer/task_coordinator.goindex 8b7ffd1419..5a3b61abe6 100644--- a/internal/syncer/task_coordinator.go+++ b/internal/syncer/task_coordinator.go@@ -47,10 +47,16 @@ type TaskCoordinator struct {pruneService *service.SyncPruneServiceidResolver *service.DocumentIDResolverexecutor *SyncJobExecutor+ checkpoints SyncCheckpointStore+}++// TaskOutcome describes post-run scheduling work for a completed task.+type TaskOutcome struct {+ NextTaskID string}// NewTaskCoordinator creates a coordinator for one claimed task at a time.-func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTaskService, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService, idResolver *service.DocumentIDResolver, executor *SyncJobExecutor) *TaskCoordinator {+func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTaskService, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService, idResolver *service.DocumentIDResolver, executor *SyncJobExecutor, checkpoints SyncCheckpointStore) *TaskCoordinator {if config.ItemRetryCount <= 0 {config.ItemRetryCount = 1}@@ -61,39 +67,44 @@ func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTif executor == nil {panic("task coordinator executor must not be nil")}+ if checkpoints == nil {+ checkpoints = newMemorySyncCheckpointStore()+ }- return &TaskCoordinator{config: config, taskService: taskService, registry: registry, sink: sink, pruneService: pruneService, idResolver: idResolver, executor: executor}+ return &TaskCoordinator{config: config, taskService: taskService, registry: registry, sink: sink, pruneService: pruneService, idResolver: idResolver, executor: executor, checkpoints: checkpoints}}// Execute dispatches a sync_logs task by task type.-func (c *TaskCoordinator) Execute(ctx context.Context, taskContext service.SyncTaskContext, lease ConnectorLockLease) error {+func (c *TaskCoordinator) Execute(ctx context.Context, taskContext service.SyncTaskContext, lease ConnectorLockLease) (TaskOutcome, error) {runCtx, cancel := context.WithDeadline(ctx, taskExecutionDeadline(time.Now(), taskContext, lease))defer cancel()ctx = runCtxconnector, err := c.registry.Open(ctx, taskContext)if err != nil {- return err+ return TaskOutcome{}, err}if err = connector.Validate(ctx); err != nil {- return err+ return TaskOutcome{}, err}switch taskContext.Task.TaskType {case service.TaskTypeSync:queue, err := c.executor.RegisterTask(ctx, taskContext.Task.ID)if err != nil {- return err+ return TaskOutcome{}, err}defer queue.Close()- runner := NewSyncRunner(c.config, c.taskService, c.sink, c.idResolver, queue)- return runner.Run(ctx, taskContext, connector)+ runner := NewSyncRunner(c.config, c.taskService, c.sink, c.idResolver, queue, c.checkpoints)+ nextTaskID, err := runner.Run(ctx, taskContext, connector)+ return TaskOutcome{NextTaskID: nextTaskID}, errcase service.TaskTypePrune:runner := NewPruneRunner(c.taskService, c.pruneService)- return runner.Run(ctx, taskContext, connector)+ nextTaskID, err := runner.Run(ctx, taskContext, connector)+ return TaskOutcome{NextTaskID: nextTaskID}, errdefault:- return fmt.Errorf("unsupported sync task type %q", taskContext.Task.TaskType)+ return TaskOutcome{}, fmt.Errorf("unsupported sync task type %q", taskContext.Task.TaskType)}}diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.goindex 044a5c17bc..caffa9ac3c 100644--- a/internal/syncer/task_worker.go+++ b/internal/syncer/task_worker.go@@ -24,6 +24,8 @@ import ("ragflow/internal/service""sync""time"++ "go.uber.org/zap")// TaskWorker consumes claimed task envelopes and runs coordinators.@@ -32,6 +34,7 @@ type TaskWorker struct {taskService *service.SyncTaskServicecoordinator *TaskCoordinatorlocker ConnectorLocker+ scheduler *Scheduler}// NewTaskWorker creates a bounded task worker pool.@@ -39,6 +42,12 @@ func NewTaskWorker(queue <-chan TaskEnvelope, taskService *service.SyncTaskServireturn &TaskWorker{queue: queue, taskService: taskService, coordinator: coordinator, locker: locker}}+// WithScheduler attaches the event scheduler used for one-shot task publishing.+func (w *TaskWorker) WithScheduler(scheduler *Scheduler) *TaskWorker {+ w.scheduler = scheduler+ return w+}+// Run starts worker goroutines and blocks until context cancellation.func (w *TaskWorker) Run(ctx context.Context, concurrency int) {var wg sync.WaitGroup@@ -107,21 +116,38 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {lease, locked := w.locker.TryLock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)if !locked {_ = w.taskService.RescheduleClaimed(ctx, taskContext.Task.ID)+ w.scheduleRetry(ctx, taskContext.Task.ID, 3*time.Second)ackEnvelope(envelope)return}defer w.locker.Unlock(taskContext.Connector.ID, taskContext.Knowledgebase.ID)startedAt := time.Now()- if err = w.coordinator.Execute(ctx, taskContext, lease); err != nil { // execute the task(sync/ prune)- logSyncTaskDuration(taskContext, startedAt)- if errors.Is(err, errSyncTaskCanceled) { // task is canceled by the user+ outcome, err := w.coordinator.Execute(ctx, taskContext, lease)+ if err != nil { // execute the task(sync/ prune)+ logSyncTaskDuration(taskContext, startedAt) // test sync task run time+ if errors.Is(err, errSyncTaskCanceled) { // task is canceled by the userackEnvelope(envelope)return}if ctx.Err() != nil { // the task is canceled by system, this need to rerun_ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID)- nackEnvelope(envelope)+ w.scheduleRetry(context.WithoutCancel(ctx), taskContext.Task.ID, 3*time.Second)+ ackEnvelope(envelope)+ return+ }+ if isTransientSyncError(err) {+ attempts, failed, transientErr := w.taskService.HandleTransientFailure(ctx, taskContext.Task.ID, taskContext.Connector.ID, err, maxTransientTaskRetries)+ if transientErr != nil {+ _ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID)+ nackEnvelope(envelope)+ return+ }+ logTransientSyncRetry(taskContext, attempts, failed, err)+ if !failed {+ w.scheduleRetry(ctx, taskContext.Task.ID, transientRetryDelay(attempts))+ }+ ackEnvelope(envelope)return}if failErr := w.taskService.Fail(ctx, taskContext.Task.ID, taskContext.Connector.ID, fmt.Errorf("sync task failed: %w", err)); failErr != nil {@@ -133,9 +159,64 @@ func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {return}logSyncTaskDuration(taskContext, startedAt) // Todo delete soon+ w.scheduleNext(ctx, outcome.NextTaskID)ackEnvelope(envelope)}+func (w *TaskWorker) scheduleNext(ctx context.Context, taskID string) {+ if w.scheduler == nil || taskID == "" {+ return+ }+ task, err := w.taskService.GetScheduledTask(ctx, taskID)+ if err != nil {+ common.Warn("syncer schedule next task lookup failed", zap.String("task_id", taskID), zap.Error(err))+ return+ }+ if err = w.scheduler.ScheduleTask(ctx, task); err != nil {+ common.Warn("syncer schedule next task failed", zap.String("task_id", taskID), zap.Error(err))+ }+}++// scheduleRetry schedule retry when web error occurred+func (w *TaskWorker) scheduleRetry(ctx context.Context, taskID string, delay time.Duration) {+ if w.scheduler == nil || taskID == "" {+ return+ }+ if err := w.scheduler.ScheduleTaskAfter(ctx, taskID, delay); err != nil {+ common.Warn("syncer retry task publish failed", zap.String("task_id", taskID), zap.Duration("delay", delay), zap.Error(err))+ }+}++// transientRetryDelay return retry delay+func transientRetryDelay(attempts int64) time.Duration {+ if attempts < 1 {+ attempts = 1+ }++ shift := attempts - 1+ if shift > 5 {+ shift = 5+ }+ return time.Duration(1<<shift) * 30 * time.Second+}++func logTransientSyncRetry(taskContext service.SyncTaskContext, attempts int64, failed bool, err error) {+ message := "sync task transient retry scheduled"+ if failed {+ message = "sync task failed after transient retries"+ }+ common.Warn(+ message,+ zap.String("task_id", taskContext.Task.ID),+ zap.String("connector_id", taskContext.Connector.ID),+ zap.String("kb_id", taskContext.Knowledgebase.ID),+ zap.String("source", taskContext.Connector.Source),+ zap.Int64("attempts", attempts),+ zap.Int64("max_retries", maxTransientTaskRetries),+ zap.Error(err),+ )+}+func ackEnvelope(envelope TaskEnvelope) {if envelope.Handle != nil {_ = envelope.Handle.Ack()diff --git a/internal/syncer/transient_error.go b/internal/syncer/transient_error.gonew file mode 100644index 0000000000..3c6047e071--- /dev/null+++ b/internal/syncer/transient_error.go@@ -0,0 +1,56 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "errors"+ "io"+ "net"+ "ragflow/internal/service"+ "strings"+)++// maxTransientTaskRetries max retry time when network error occur+const maxTransientTaskRetries int64 = 3++// isTransientSyncError reports whether a task-level sync error is worth retrying.+func isTransientSyncError(err error) bool {+ if err == nil {+ return false+ }+ if service.IsRetryable(err) || errors.Is(err, io.ErrUnexpectedEOF) {+ return true+ }++ var netErr net.Error+ if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) {+ return true+ }++ message := strings.ToLower(err.Error())+ return strings.Contains(message, "unexpected eof") ||+ strings.Contains(message, "connection reset") ||+ strings.Contains(message, "connection refused") ||+ strings.Contains(message, "timeout") ||+ strings.Contains(message, "temporary") ||+ strings.Contains(message, "too many requests") ||+ strings.Contains(message, "http 429") ||+ strings.Contains(message, "http 500") ||+ strings.Contains(message, "http 502") ||+ strings.Contains(message, "http 503") ||+ strings.Contains(message, "http 504")+}