Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat[Go]: complete the base for data Syncer (#17890)

Haruko386f532f27f1f36
diff --git a/internal/dao/sync_prune_snapshot.go b/internal/dao/sync_prune_snapshot.go
new file mode 100644
index 0000000000..bf4d4a964f
--- /dev/null
+++ b/internal/dao/sync_prune_snapshot.go
@@ -0,0 +1,69 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ syncerconnector "ragflow/internal/syncer/connector"
+
+ "gorm.io/gorm"
+)
+
+// SyncPruneSnapshot stores one source ID from a complete prune snapshot.
+type SyncPruneSnapshot struct {
+ TaskID string `gorm:"column:task_id;primaryKey;size:32"`
+ SourceDocID string `gorm:"column:source_doc_id;primaryKey;size:512"`
+}
+
+// TableName returns the prune snapshot table name.
+func (SyncPruneSnapshot) TableName() string {
+ return "sync_prune_snapshot"
+}
+
+// SyncPruneSnapshotDAO manages temporary prune snapshots.
+type SyncPruneSnapshotDAO struct {
+ db *gorm.DB
+}
+
+// NewSyncPruneSnapshotDAO creates a prune snapshot DAO.
+func NewSyncPruneSnapshotDAO(db *gorm.DB) *SyncPruneSnapshotDAO {
+ return &SyncPruneSnapshotDAO{db: db}
+}
+
+// InsertBatch writes one prune snapshot batch.
+func (d *SyncPruneSnapshotDAO) InsertBatch(ctx context.Context, taskID string, documents []syncerconnector.SlimDocument) error {
+ if len(documents) == 0 {
+ return nil
+ }
+ rows := make([]SyncPruneSnapshot, 0, len(documents))
+ for _, doc := range documents {
+ rows = append(rows, SyncPruneSnapshot{TaskID: taskID, SourceDocID: doc.SourceID})
+ }
+ return d.db.WithContext(ctx).Create(&rows).Error
+}
+
+// ListSourceIDs returns all source IDs for a task snapshot.
+func (d *SyncPruneSnapshotDAO) ListSourceIDs(ctx context.Context, taskID string) ([]string, error) {
+ var ids []string
+ err := d.db.WithContext(ctx).Model(&SyncPruneSnapshot{}).Where("task_id = ?", taskID).Pluck("source_doc_id", &ids).Error
+ return ids, err
+}
+
+// Clear removes all snapshot rows for a task.
+func (d *SyncPruneSnapshotDAO) Clear(ctx context.Context, taskID string) error {
+ return d.db.WithContext(ctx).Where("task_id = ?", taskID).Delete(&SyncPruneSnapshot{}).Error
+}
diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.go
new file mode 100644
index 0000000000..abb0961a57
--- /dev/null
+++ b/internal/dao/sync_task.go
@@ -0,0 +1,376 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "errors"
+ "ragflow/internal/entity"
+ "ragflow/internal/utility"
+ "time"
+
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+const (
+ // TaskTypeSync is the Python-compatible SYNC task type.
+ TaskTypeSync = "sync"
+ // TaskTypePrune is the Python-compatible PRUNE task type.
+ TaskTypePrune = "prune"
+ // SyncStatusSchedule is the Python TaskStatus.SCHEDULE value for sync_logs.
+ SyncStatusSchedule = "5"
+ // SyncStatusRunning is the Python TaskStatus.RUNNING value for sync_logs.
+ SyncStatusRunning = "1"
+ // SyncStatusCancel is the Python TaskStatus.CANCEL value for sync_logs.
+ SyncStatusCancel = "2"
+ // SyncStatusDone is the Python TaskStatus.DONE value for sync_logs.
+ SyncStatusDone = "3"
+ // SyncStatusFail is the Python TaskStatus.FAIL value for sync_logs.
+ SyncStatusFail = "4"
+)
+
+// SyncTask is the Python-compatible sync task row.
+type SyncTask struct {
+ entity.SyncLogs `gorm:"embedded"`
+}
+
+// TableName returns the Python-compatible sync task table.
+func (SyncTask) TableName() string {
+ return "sync_logs"
+}
+
+// SyncTaskContext contains every database row required to execute one task.
+type SyncTaskContext struct {
+ Task entity.SyncLogs
+ Connector entity.Connector
+ Connector2Kb entity.Connector2Kb
+ Knowledgebase entity.Knowledgebase
+}
+
+// ConnectorSource returns the connector source name.
+func (c SyncTaskContext) ConnectorSource() string {
+ return c.Connector.Source
+}
+
+// SyncTaskDAO reads and updates sync_logs tasks.
+type SyncTaskDAO struct {
+ db *gorm.DB
+}
+
+// NewSyncTaskDAO creates a syncer task DAO.
+func NewSyncTaskDAO(db *gorm.DB) *SyncTaskDAO {
+ if db == nil {
+ db = GetDB()
+ }
+ return &SyncTaskDAO{db: db}
+}
+
+// DB returns the DAO database handle.
+func (d *SyncTaskDAO) DB() *gorm.DB {
+ return d.db
+}
+
+type dueSyncTaskRow struct {
+ entity.SyncLogs
+ ConnectorRefreshFreq int64 `gorm:"column:connector_refresh_freq"`
+ ConnectorPruneFreq int64 `gorm:"column:connector_prune_freq"`
+ ConnectorConfig entity.JSONMap `gorm:"column:connector_config"`
+}
+
+// ListDueTasks returns due schedule tasks across SYNC and PRUNE.
+func (d *SyncTaskDAO) ListDueTasks(ctx context.Context, now time.Time, limit int) ([]entity.SyncLogs, error) {
+ var rows []dueSyncTaskRow
+ if err := d.db.WithContext(ctx).
+ Model(&entity.SyncLogs{}).
+ Select("sync_logs.*, connector.refresh_freq AS connector_refresh_freq, connector.prune_freq AS connector_prune_freq, connector.config AS connector_config").
+ Joins("JOIN connector ON sync_logs.connector_id = connector.id").
+ Joins("JOIN connector2kb ON sync_logs.connector_id = connector2kb.connector_id AND sync_logs.kb_id = connector2kb.kb_id").
+ Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").
+ Where("sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).
+ Order("sync_logs.update_time DESC").
+ Limit(limit * 4).
+ Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+
+ tasks := make([]entity.SyncLogs, 0, limit)
+ for _, row := range rows {
+ if !isDue(row.SyncLogs, row.ConnectorRefreshFreq, row.ConnectorPruneFreq, row.ConnectorConfig, now) {
+ continue
+ }
+ tasks = append(tasks, row.SyncLogs)
+ if len(tasks) >= limit {
+ break
+ }
+ }
+ return tasks, nil
+}
+
+// ClaimTask conditionally marks a scheduled task as running.
+func (d *SyncTaskDAO) ClaimTask(ctx context.Context, taskID string, now time.Time) (bool, error) {
+ result := d.db.WithContext(ctx).Model(&entity.SyncLogs{}).
+ Where("id = ? AND status = ?", taskID, SyncStatusSchedule).
+ Updates(map[string]any{"status": SyncStatusRunning, "time_started": now})
+ if result.Error != nil {
+ return false, result.Error
+ }
+ return result.RowsAffected == 1, nil
+}
+
+// GetTaskContext loads a task with connector, mapping, and knowledgebase rows.
+func (d *SyncTaskDAO) GetTaskContext(ctx context.Context, taskID string) (SyncTaskContext, error) {
+ var task entity.SyncLogs
+ if err := d.db.WithContext(ctx).Where("id = ?", taskID).First(&task).Error; err != nil {
+ return SyncTaskContext{}, err
+ }
+ // get connector
+ var connector entity.Connector
+ if err := d.db.WithContext(ctx).Where("id = ?", task.ConnectorID).First(&connector).Error; err != nil {
+ return SyncTaskContext{}, err
+ }
+ // get relation
+ var connector2Kb entity.Connector2Kb
+ if err := d.db.WithContext(ctx).Where("connector_id = ? AND kb_id = ?", task.ConnectorID, task.KbID).First(&connector2Kb).Error; err != nil {
+ return SyncTaskContext{}, err
+ }
+ // get KB
+ var kb entity.Knowledgebase
+ if err := d.db.WithContext(ctx).Where("id = ?", task.KbID).First(&kb).Error; err != nil {
+ return SyncTaskContext{}, err
+ }
+
+ return SyncTaskContext{Task: task, Connector: connector, Connector2Kb: connector2Kb, Knowledgebase: kb}, nil
+}
+
+// MarkConnectorRunning marks a connector running.
+func (d *SyncTaskDAO) MarkConnectorRunning(ctx context.Context, connectorID string) error {
+ return d.db.WithContext(ctx).Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusRunning).Error
+}
+
+// RescheduleClaimed puts a claimed task back into schedule state.
+func (d *SyncTaskDAO) RescheduleClaimed(ctx context.Context, taskID string) error {
+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var task entity.SyncLogs
+ if err := tx.WithContext(ctx).Where("id = ? AND status = ?", taskID, SyncStatusRunning).First(&task).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil
+ }
+ return err
+ }
+ if err := tx.Model(&entity.SyncLogs{}).
+ Where("id = ? AND status = ?", taskID, SyncStatusRunning).
+ Update("status", SyncStatusSchedule).Error; err != nil {
+ return err
+ }
+ return tx.Model(&entity.Connector{}).Where("id = ? AND status = ?", task.ConnectorID, SyncStatusRunning).Update("status", SyncStatusSchedule).Error
+ })
+}
+
+// FailTask marks a task failed without advancing its poll waterline.
+func (d *SyncTaskDAO) FailTask(ctx context.Context, taskID, connectorID, message string, errorCount int64) error {
+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Model(&entity.SyncLogs{}).Where("id = ?", taskID).Updates(map[string]any{
+ "status": SyncStatusFail,
+ "error_msg": message,
+ "error_count": errorCount,
+ }).Error; err != nil {
+ return err
+ }
+ if connectorID == "" {
+ return nil
+ }
+ return tx.Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusFail).Error
+ })
+}
+
+// CompleteSyncTask marks SYNC done and creates the next schedule task.
+func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, newDocs, totalDocs, errorCount int64, errorMsg string) error {
+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Model(&entity.SyncLogs{}).Where("id = ?", taskContext.Task.ID).Updates(map[string]any{
+ "status": SyncStatusDone,
+ "poll_range_end": pollRangeEnd,
+ "new_docs_indexed": newDocs,
+ "total_docs_indexed": gorm.Expr("total_docs_indexed + ?", totalDocs),
+ "error_msg": errorMsg,
+ "error_count": errorCount,
+ }).Error; err != nil {
+ return err
+ }
+ if err := tx.Model(&entity.Connector{}).Where("id = ?", taskContext.Connector.ID).Update("status", SyncStatusDone).Error; err != nil {
+ return err
+ }
+
+ return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypeSync, false, &pollRangeEnd, taskContext.Task.TotalDocsIndexed+totalDocs)
+ })
+}
+
+// CompletePruneTask marks PRUNE done and creates the next schedule task.
+func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTaskContext, removed int64) error {
+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Model(&entity.SyncLogs{}).Where("id = ?", taskContext.Task.ID).Updates(map[string]any{
+ "status": SyncStatusDone,
+ "docs_removed_from_index": gorm.Expr("docs_removed_from_index + ?", removed),
+ }).Error; err != nil {
+ return err
+ }
+ if err := tx.Model(&entity.Connector{}).Where("id = ?", taskContext.Connector.ID).Update("status", SyncStatusDone).Error; err != nil {
+ return err
+ }
+ if !syncConnectorConfigBool(taskContext.Connector.Config, "sync_deleted_files") {
+ return nil
+ }
+ return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypePrune, false, nil, taskContext.Task.TotalDocsIndexed)
+ })
+}
+
+// RecoverStaleRunning restores timed-out running tasks to schedule.
+func (d *SyncTaskDAO) RecoverStaleRunning(ctx context.Context, now time.Time) (int64, error) {
+ type staleRunningTaskRow struct {
+ ID string `gorm:"column:id"`
+ ConnectorID string `gorm:"column:connector_id"`
+ TimeStarted *time.Time `gorm:"column:time_started"`
+ ConnectorTimeoutSecs int64 `gorm:"column:connector_timeout_secs"`
+ }
+ var rows []staleRunningTaskRow
+ if err := d.db.WithContext(ctx).
+ Model(&entity.SyncLogs{}).
+ Select("sync_logs.id, sync_logs.connector_id, sync_logs.time_started, connector.timeout_secs AS connector_timeout_secs").
+ Joins("JOIN connector ON sync_logs.connector_id = connector.id").
+ Where("sync_logs.status = ?", SyncStatusRunning).
+ Scan(&rows).Error; err != nil {
+ return 0, err
+ }
+ var recovered int64
+ connectorIDs := map[string]struct{}{}
+
+ for _, row := range rows {
+ if row.TimeStarted == nil {
+ continue
+ }
+ timeout := time.Duration(row.ConnectorTimeoutSecs) * time.Second
+ if timeout <= 0 {
+ timeout = time.Hour
+ }
+ if row.TimeStarted.Add(timeout).After(now) {
+ continue
+ }
+ if err := d.RescheduleClaimed(ctx, row.ID); err != nil {
+ return recovered, err
+ }
+ recovered++
+ connectorIDs[row.ConnectorID] = struct{}{}
+ }
+
+ if recovered == 0 {
+ return 0, nil
+ }
+ ids := make([]string, 0, len(connectorIDs))
+
+ for connectorID := range connectorIDs {
+ ids = append(ids, connectorID)
+ }
+
+ return recovered, d.db.WithContext(ctx).Model(&entity.Connector{}).Where("id IN ? AND status = ?", ids, SyncStatusRunning).Update("status", SyncStatusSchedule).Error
+}
+
+// createScheduledTask creates the next Python-compatible scheduled task.
+func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taskType string, fromBeginning bool, pollRangeStart *time.Time, totalDocsIndexed int64) error {
+ var lockRow entity.Connector2Kb
+ query := tx.WithContext(ctx)
+ if tx.Dialector.Name() != "sqlite" {
+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})
+ }
+ if err := query.
+ Where("connector_id = ? AND kb_id = ?", connectorID, kbID).
+ First(&lockRow).Error; err != nil {
+ return err
+ }
+
+ var existing int64
+ if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).
+ Where("connector_id = ? AND kb_id = ? AND task_type = ? AND status = ?", connectorID, kbID, taskType, SyncStatusSchedule).
+ Count(&existing).Error; err != nil {
+ return err
+ }
+ if existing > 0 {
+ return nil
+ }
+
+ reindex := "0"
+ if fromBeginning {
+ reindex = "1"
+ }
+
+ now := time.Now().Local()
+ return tx.WithContext(ctx).Create(&entity.SyncLogs{
+ ID: utility.GenerateToken(),
+ ConnectorID: connectorID,
+ KbID: kbID,
+ TaskType: taskType,
+ Status: SyncStatusSchedule,
+ FromBeginning: &reindex,
+ PollRangeStart: pollRangeStart,
+ TimeStarted: &now,
+ ErrorMsg: "",
+ TotalDocsIndexed: totalDocsIndexed,
+ }).Error
+}
+
+// isDue applies refresh_freq and prune_freq scheduling semantics.
+func isDue(task entity.SyncLogs, refreshFreq, pruneFreq int64, config map[string]any, now time.Time) bool {
+ if task.UpdateDate == nil {
+ return true
+ }
+ var freqMinutes int64
+ switch task.TaskType {
+ case TaskTypeSync:
+ freqMinutes = refreshFreq
+ case TaskTypePrune:
+ if !syncConnectorConfigBool(config, "sync_deleted_files") {
+ return false
+ }
+ freqMinutes = pruneFreq
+ default:
+ return false
+ }
+ if freqMinutes <= 0 {
+ return true
+ }
+ return task.UpdateDate.Before(now.Add(-time.Duration(freqMinutes) * time.Minute))
+}
+
+// syncConnectorConfigBool reads a Python JSON bool/string flag.
+func syncConnectorConfigBool(config map[string]any, key string) bool {
+ value, ok := config[key]
+ if !ok {
+ return false
+ }
+ switch typed := value.(type) {
+ case bool:
+ return typed
+ case string:
+ return typed == "1" || typed == "true" || typed == "TRUE"
+ default:
+ return false
+ }
+}
+
+// IsNotFound reports whether an error is a gorm not found error.
+func IsNotFound(err error) bool {
+ return errors.Is(err, gorm.ErrRecordNotFound)
+}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 2a69b61b3d..8df895966b 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -40,6 +40,8 @@ import (
"ragflow/internal/service"
"ragflow/internal/service/file"
"ragflow/internal/storage"
+ syncerconnector "ragflow/internal/syncer/connector"
+ "ragflow/internal/utility"
)
// recordingTaskPublisher implements service.TaskPublisher and records published messages.
@@ -55,7 +57,8 @@ func (r *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.T
}
type fakeUploadStorage struct {
- objects map[string][]byte
+ objects map[string][]byte
+ afterPut func()
}
func newFakeUploadStorage() *fakeUploadStorage {
@@ -67,6 +70,9 @@ func (f *fakeUploadStorage) Health(_ context.Context) bool { return true }
func (f *fakeUploadStorage) key(bucket, fnm string) string { return bucket + "/" + fnm }
func (f *fakeUploadStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
f.objects[f.key(bucket, fnm)] = append([]byte(nil), binary...)
+ if f.afterPut != nil {
+ f.afterPut()
+ }
return nil
}
func (f *fakeUploadStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
@@ -730,6 +736,121 @@ func TestContentHashHex_MatchesPythonXXH128(t *testing.T) {
}
}
+func TestSyncDocumentUpsertRemovesStagedBlobWhenInsertFails(t *testing.T) {
+ ctx := context.Background()
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ mockStorage := newFakeUploadStorage()
+ factory := storage.GetStorageFactory()
+ origStorage := factory.GetStorage()
+ factory.SetStorage(mockStorage)
+ t.Cleanup(func() { factory.SetStorage(origStorage) })
+
+ svc := testDocumentService(t)
+ _, err := svc.Upsert(ctx, service.DocumentUpsertInput{
+ TaskContext: service.SyncTaskContext{
+ Connector: entity.Connector{TenantID: "tenant-1"},
+ Knowledgebase: entity.Knowledgebase{
+ ID: "missing-kb",
+ TenantID: "tenant-1",
+ Name: "Missing KB",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ },
+ },
+ SourceType: "github",
+ DocumentID: "doc-sync-insert",
+ SourceDocument: syncerconnector.SourceDocument{
+ SourceID: "source-1",
+ SemanticIdentifier: "source",
+ Extension: ".txt",
+ Blob: []byte("new content"),
+ },
+ })
+ if err == nil {
+ t.Fatalf("expected insert failure")
+ }
+ for key := range mockStorage.objects {
+ if strings.Contains(key, "/.staged/") {
+ t.Fatalf("expected staged object cleanup, found %s", key)
+ }
+ }
+}
+
+func TestSyncDocumentUpsertRemovesStagedBlobWhenUpdateFails(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ mockStorage := newFakeUploadStorage()
+ factory := storage.GetStorageFactory()
+ origStorage := factory.GetStorage()
+ factory.SetStorage(mockStorage)
+ t.Cleanup(func() { factory.SetStorage(origStorage) })
+
+ activeLocation := "sync/github/doc-sync-update.txt"
+ if err := mockStorage.Put(context.Background(), "kb-sync", activeLocation, []byte("old content")); err != nil {
+ t.Fatalf("seed active object: %v", err)
+ }
+ oldName := "old.txt"
+ oldHash := "old-hash"
+ if err := db.Create(&entity.Document{
+ ID: "doc-sync-update",
+ KbID: "kb-sync",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "github",
+ Type: string(utility.FileTypeTXT),
+ CreatedBy: "tenant-1",
+ Name: &oldName,
+ Location: &activeLocation,
+ Size: int64(len("old content")),
+ Suffix: "txt",
+ ContentHash: &oldHash,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ svc := testDocumentService(t)
+ mockStorage.afterPut = cancel
+ _, err := svc.Upsert(ctx, service.DocumentUpsertInput{
+ TaskContext: service.SyncTaskContext{
+ Connector: entity.Connector{TenantID: "tenant-1"},
+ Knowledgebase: entity.Knowledgebase{
+ ID: "kb-sync",
+ TenantID: "tenant-1",
+ Name: "Sync KB",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ },
+ },
+ SourceType: "github",
+ DocumentID: "doc-sync-update",
+ SourceDocument: syncerconnector.SourceDocument{
+ SourceID: "source-1",
+ SemanticIdentifier: "source",
+ Extension: ".txt",
+ Blob: []byte("new content"),
+ },
+ })
+ if err == nil {
+ t.Fatalf("expected update failure")
+ }
+ got, err := mockStorage.Get(context.Background(), "kb-sync", activeLocation)
+ if err != nil {
+ t.Fatalf("active object was removed: %v", err)
+ }
+ if string(got) != "old content" {
+ t.Fatalf("active object overwritten: %q", got)
+ }
+ for key := range mockStorage.objects {
+ if strings.Contains(key, "/.staged/") {
+ t.Fatalf("expected staged object cleanup, found %s", key)
+ }
+ }
+}
+
func TestUploadLocalDocuments_MirrorsPythonCoreFields(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/document/sync_document.go b/internal/service/document/sync_document.go
new file mode 100644
index 0000000000..56ab4a5963
--- /dev/null
+++ b/internal/service/document/sync_document.go
@@ -0,0 +1,315 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package document
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/service"
+ "ragflow/internal/storage"
+ "ragflow/internal/utility"
+
+ "gorm.io/gorm"
+)
+
+var syncFilenameUnsafeRE = regexp.MustCompile(`[\\/:*?"<>|]+`)
+
+// Upsert stores one normalized source document.
+func (s *DocumentService) Upsert(ctx context.Context, input service.DocumentUpsertInput) (service.DocumentUpsertResult, error) {
+ if input.DocumentID == "" {
+ return service.DocumentUpsertResult{}, fmt.Errorf("document id is empty")
+ }
+
+ // get base info
+ kb := &input.TaskContext.Knowledgebase
+ tenantID := input.TaskContext.Connector.TenantID
+ // return normalized filename
+ filename := syncDocumentFilename(input.SourceDocument.SemanticIdentifier, input.SourceDocument.Extension, input.SourceDocument.SourceID)
+ filetype := utility.FilenameType(filename)
+ if filetype == utility.FileTypeOTHER {
+ return service.DocumentUpsertResult{}, fmt.Errorf("document type %q is not supported", filename)
+ }
+
+ // get storage
+ storageImpl := storage.GetStorageFactory().GetStorage()
+ if storageImpl == nil {
+ return service.DocumentUpsertResult{}, fmt.Errorf("storage not initialized")
+ }
+
+ // get content hash
+ contentHash := input.SourceDocument.Fingerprint
+ if contentHash == "" {
+ contentHash = contentHashHex(input.SourceDocument.Blob)
+ }
+
+ // if the 'file' is existing
+ existing, err := s.documentDAO.GetByID(ctx, dao.DB, input.DocumentID)
+ if err != nil && !errorsIsRecordNotFound(err) {
+ return service.DocumentUpsertResult{}, err
+ }
+ if existing != nil && existing.ContentHash != nil && *existing.ContentHash == contentHash {
+ if err := s.ensureSyncDocumentPostWrite(ctx, input, existing); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ return service.DocumentUpsertResult{DocID: input.DocumentID, Action: service.DocumentActionSkipped}, nil
+ }
+
+ // store location
+ location := syncDocumentStagedLocation(input.SourceType, input.DocumentID, filename)
+ cleanupLocation := true
+ defer func() {
+ if cleanupLocation {
+ _ = storageImpl.Remove(context.WithoutCancel(ctx), kb.ID, location)
+ }
+ }()
+ if err = storageImpl.Put(ctx, kb.ID, location, input.SourceDocument.Blob); err != nil {
+ cleanupLocation = false
+ return service.DocumentUpsertResult{}, err
+ }
+
+ // not existing, insert a new 'file'
+ if existing == nil {
+ result, insertErr := s.insertSyncDocument(ctx, input, filename, location, string(filetype), contentHash, tenantID)
+ if insertErr == nil {
+ cleanupLocation = false
+ }
+ return result, insertErr
+ }
+ // just update the file
+ result, updateErr := s.updateSyncDocument(ctx, input, existing, filename, location, string(filetype), contentHash)
+ if updateErr == nil {
+ cleanupLocation = false
+ }
+ return result, updateErr
+}
+
+// insertSyncDocument inserts a new synced document and file-manager link.
+func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.DocumentUpsertInput, filename, location, filetype, contentHash, tenantID string) (service.DocumentUpsertResult, error) {
+ // base info
+ kb := &input.TaskContext.Knowledgebase
+ kbFolder, err := s.ensureKBFolder(ctx, kb, tenantID)
+ if err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+
+ // create the 'doc'
+ doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, copyJSONMap(kb.ParserConfig), input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)
+ doc.ID = input.DocumentID
+
+ // put 'file' in mysql `document`
+ if err = s.InsertDocument(doc); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ if err = s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {
+ return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)
+ }
+ // link 'file' 2 'file_system'
+ if err = s.addFileFromKB(ctx, doc, kbFolder.ID, kb.TenantID); err != nil {
+ return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)
+ }
+ // write metadata and do `auto_parse` (or not)
+ if err = s.afterSyncDocumentUpsert(ctx, input, doc, false); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+
+ return service.DocumentUpsertResult{DocID: doc.ID, Action: service.DocumentActionAdded}, nil
+}
+
+// updateSyncDocument updates an existing synced document and file-manager row.
+func (s *DocumentService) updateSyncDocument(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, filename, location, filetype, contentHash string) (service.DocumentUpsertResult, error) {
+ suffix := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".")
+ updates := map[string]any{
+ "name": filename,
+ "size": int64(len(input.SourceDocument.Blob)),
+ "type": filetype,
+ "suffix": suffix,
+ "source_type": input.SourceType,
+ }
+ if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, updates); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+
+ doc.Name = &filename
+ doc.Size = int64(len(input.SourceDocument.Blob))
+ doc.Type = filetype
+ doc.Suffix = suffix
+ doc.SourceType = input.SourceType
+
+ if err := s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ // update new updated file
+ if err := s.updateSyncDocumentFile(ctx, input, doc); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ // write metadata and do `auto_parse`
+ if err := s.afterSyncDocumentUpsert(ctx, input, doc, true); err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ return service.DocumentUpsertResult{DocID: doc.ID, Action: service.DocumentActionUpdated}, nil
+}
+
+// publishSyncDocument publishes the staged object key after the document row exists.
+func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.Document, location, contentHash string) error {
+ if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
+ "location": location,
+ "content_hash": contentHash,
+ }); err != nil {
+ return err
+ }
+ doc.Location = &location
+ doc.ContentHash = &contentHash
+ return nil
+}
+
+// afterSyncDocumentUpsert writes metadata and optionally enqueues parsing.
+func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, rerun bool) error {
+ // write metadata
+ if len(input.SourceDocument.Metadata) > 0 && s.docEngine != nil {
+ if err := s.SetDocumentMetadata(ctx, doc.ID, input.SourceDocument.Metadata); err != nil {
+ return err
+ }
+ }
+ // do auto_parse if enabled
+ if !input.AutoParse {
+ return nil
+ }
+ return s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{RerunWithDelete: rerun})
+}
+
+// ensureSyncDocumentPostWrite retries dependent work before an unchanged document is skipped.
+func (s *DocumentService) ensureSyncDocumentPostWrite(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document) error {
+ if err := s.updateSyncDocumentFile(ctx, input, doc); err != nil {
+ return err
+ }
+ if len(input.SourceDocument.Metadata) > 0 && s.docEngine != nil {
+ if err := s.SetDocumentMetadata(ctx, doc.ID, input.SourceDocument.Metadata); err != nil {
+ return err
+ }
+ }
+ if !input.AutoParse {
+ return nil
+ }
+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return err
+ }
+ if task != nil {
+ return nil
+ }
+ return s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{})
+}
+
+// updateSyncDocumentFile updates the file-manager row linked to a synced document.
+func (s *DocumentService) updateSyncDocumentFile(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document) error {
+ mappings, err := s.file2DocumentDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return err
+ }
+
+ // no updates
+ if len(mappings) == 0 || mappings[0].FileID == nil {
+ kbFolder, folderErr := s.ensureKBFolder(ctx, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID)
+ if folderErr != nil {
+ return folderErr
+ }
+ return s.addFileFromKB(ctx, doc, kbFolder.ID, input.TaskContext.Knowledgebase.TenantID)
+ }
+
+ name := ""
+ if doc.Name != nil {
+ name = *doc.Name
+ }
+
+ location := ""
+ if doc.Location != nil {
+ location = *doc.Location
+ }
+
+ return s.fileDAO.UpdateByID(ctx, dao.DB, *mappings[0].FileID, map[string]interface{}{
+ "name": name,
+ "location": location,
+ "size": doc.Size,
+ "type": doc.Type,
+ })
+}
+
+// syncDocumentFilename returns a bounded filename for a synced source document.
+func syncDocumentFilename(name, extension, fallback string) string {
+ name = strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(name, "_"))
+ if name == "" {
+ name = strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(fallback, "_"))
+ }
+ if name == "" {
+ name = "document"
+ }
+
+ extension = strings.TrimSpace(extension)
+ if extension == "" {
+ extension = ".txt"
+ }
+
+ if !strings.HasPrefix(extension, ".") {
+ extension = "." + extension
+ }
+ if !strings.HasSuffix(strings.ToLower(name), strings.ToLower(extension)) {
+ name += extension
+ }
+
+ if len(name) <= 255 {
+ return name
+ }
+
+ ext := filepath.Ext(name)
+ baseLimit := 255 - len(ext)
+ if baseLimit < 1 {
+ return name[:255]
+ }
+
+ return name[:baseLimit] + ext
+}
+
+// syncDocumentStagedLocation returns a unique object-storage key for unpublished synced content.
+func syncDocumentStagedLocation(sourceType, docID, filename string) string {
+ ext := filepath.Ext(filename)
+ sourceType = strings.Trim(strings.ReplaceAll(sourceType, "/", "_"), "_")
+ if sourceType == "" {
+ sourceType = "sync"
+ }
+ return fmt.Sprintf("sync/%s/.staged/%s/%s%s", sourceType, utility.GenerateToken(), docID, ext)
+}
+
+// copyJSONMap returns a shallow copy of a JSON map.
+func copyJSONMap(value entity.JSONMap) entity.JSONMap {
+ out := entity.JSONMap{}
+ for k, v := range value {
+ out[k] = v
+ }
+ return out
+}
+
+// errorsIsRecordNotFound reports whether an error is GORM's not-found error.
+func errorsIsRecordNotFound(err error) bool {
+ return errors.Is(err, gorm.ErrRecordNotFound)
+}
diff --git a/internal/service/sync_document_id_resolver.go b/internal/service/sync_document_id_resolver.go
new file mode 100644
index 0000000000..b9605badc7
--- /dev/null
+++ b/internal/service/sync_document_id_resolver.go
@@ -0,0 +1,121 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "encoding/hex"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+
+ "github.com/zeebo/xxh3"
+)
+
+// Hash128 api.utils.common.hash128.
+func Hash128(data string) string {
+ sum := xxh3.Hash128([]byte(data)).Bytes()
+ return hex.EncodeToString(sum[:])
+}
+
+// ResolvedDocumentID contains legacy and new connector document IDs.
+type ResolvedDocumentID struct {
+ DocID string
+ LegacyID string
+ NewID string
+ StoredFingerprint string
+}
+
+// DocumentStore reads existing connector documents.
+type DocumentStore interface {
+ ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error)
+ GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error)
+}
+
+// DocumentIDResolver applies Python-compatible legacy/new ID rules.
+type DocumentIDResolver struct {
+ store DocumentStore
+}
+
+// NewDocumentIDResolver creates a document ID resolver.
+func NewDocumentIDResolver(store DocumentStore) *DocumentIDResolver {
+ return &DocumentIDResolver{store: store}
+}
+
+// Resolve chooses the legacy ID when already present in the KB.
+func (r *DocumentIDResolver) Resolve(ctx context.Context, kbID, connectorID, sourceType, sourceID string) (ResolvedDocumentID, error) {
+ legacyID := Hash128(connectorID + ":" + sourceID)
+ newID := Hash128(kbID + ":" + connectorID + ":" + sourceID)
+
+ fingerprints, err := r.store.GetFingerprintsByIDs(ctx, kbID, sourceType, []string{legacyID, newID})
+ if err != nil {
+ return ResolvedDocumentID{}, err
+ }
+
+ docID := newID
+ if _, ok := fingerprints[legacyID]; ok {
+ docID = legacyID
+ }
+ stored := fingerprints[legacyID]
+ if stored == "" {
+ stored = fingerprints[newID]
+ }
+
+ return ResolvedDocumentID{DocID: docID, LegacyID: legacyID, NewID: newID, StoredFingerprint: stored}, nil
+}
+
+// GormDocumentStore reads documents from the current GORM database.
+type GormDocumentStore struct{}
+
+// NewGormDocumentStore creates a GORM-backed document store.
+func NewGormDocumentStore() *GormDocumentStore {
+ return &GormDocumentStore{}
+}
+
+// ListIDs returns existing connector document IDs in a KB.
+func (s *GormDocumentStore) ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error) {
+ var docs []entity.Document
+ if err := dao.GetDB().WithContext(ctx).Select("id").Where("kb_id = ? AND source_type = ?", kbID, sourceType).Find(&docs).Error; err != nil {
+ return nil, err
+ }
+
+ result := make(map[string]struct{}, len(docs))
+ for _, doc := range docs {
+ result[doc.ID] = struct{}{}
+ }
+
+ return result, nil
+}
+
+// GetFingerprintsByIDs returns fingerprints for the requested candidate IDs.
+func (s *GormDocumentStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {
+ var docs []entity.Document
+ if err := dao.GetDB().WithContext(ctx).
+ Select("id", "content_hash").
+ Where("kb_id = ? AND source_type = ? AND id IN ?", kbID, sourceType, ids).
+ Find(&docs).Error; err != nil {
+ return nil, err
+ }
+ result := make(map[string]string, len(docs))
+ for _, doc := range docs {
+ if doc.ContentHash == nil {
+ result[doc.ID] = ""
+ continue
+ }
+ result[doc.ID] = *doc.ContentHash
+ }
+ return result, nil
+}
diff --git a/internal/service/sync_document_sink.go b/internal/service/sync_document_sink.go
new file mode 100644
index 0000000000..d40bff72ed
--- /dev/null
+++ b/internal/service/sync_document_sink.go
@@ -0,0 +1,88 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "errors"
+ syncerconnector "ragflow/internal/syncer/connector"
+ "time"
+)
+
+const (
+ // DocumentActionAdded means a document was newly inserted.
+ DocumentActionAdded = "added"
+ // DocumentActionUpdated means a document was changed.
+ DocumentActionUpdated = "updated"
+ // DocumentActionSkipped means a document was unchanged.
+ DocumentActionSkipped = "skipped"
+)
+
+// DocumentUpsertInput describes one normalized source document write.
+type DocumentUpsertInput struct {
+ TaskContext SyncTaskContext
+ SourceType string
+ DocumentID string
+ LegacyID string
+ NewID string
+ SourceDocument syncerconnector.SourceDocument
+ AutoParse bool
+}
+
+// DocumentUpsertResult describes one sink write result.
+type DocumentUpsertResult struct {
+ DocID string
+ Action string
+}
+
+// DocumentSink stores one normalized source document.
+type DocumentSink interface {
+ // Upsert stores one normalized source document.
+ Upsert(ctx context.Context, input DocumentUpsertInput) (DocumentUpsertResult, error)
+}
+
+// RetryableError marks a failure as safe to retry.
+type RetryableError struct {
+ Err error
+ After time.Duration
+ Temporary bool
+ StatusCode int
+}
+
+// Error returns the wrapped error message.
+func (e *RetryableError) Error() string {
+ if e == nil || e.Err == nil {
+ return "retryable sync error"
+ }
+ return e.Err.Error()
+}
+
+// Unwrap returns the wrapped error.
+func (e *RetryableError) Unwrap() error {
+ if e == nil {
+ return nil
+ }
+ return e.Err
+}
+
+// IsRetryable reports whether an error should be retried.
+func IsRetryable(err error) bool {
+ if _, ok := errors.AsType[*RetryableError](err); ok {
+ return true
+ }
+ return false
+}
diff --git a/internal/service/sync_prune_service.go b/internal/service/sync_prune_service.go
new file mode 100644
index 0000000000..107464ee28
--- /dev/null
+++ b/internal/service/sync_prune_service.go
@@ -0,0 +1,99 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "errors"
+ "ragflow/internal/dao"
+ syncerconnector "ragflow/internal/syncer/connector"
+)
+
+// ErrSyncDocumentDeleterNotConfigured reports a missing production delete service.
+var ErrSyncDocumentDeleterNotConfigured = errors.New("sync document deleter is not configured")
+
+// SyncDocumentDeleter removes documents through the owning business service.
+type SyncDocumentDeleter interface {
+ // DeleteDocument removes a document and dependent business state.
+ DeleteDocument(ctx context.Context, docID string) error
+}
+
+// SyncPruneService owns prune snapshots and stale document deletion.
+type SyncPruneService struct {
+ snapshotDAO *dao.SyncPruneSnapshotDAO
+ deleter SyncDocumentDeleter
+ store DocumentStore
+}
+
+// NewSyncPruneService creates a prune service.
+func NewSyncPruneService(snapshotDAO *dao.SyncPruneSnapshotDAO, deleter SyncDocumentDeleter, store DocumentStore) *SyncPruneService {
+ if store == nil {
+ store = NewGormDocumentStore()
+ }
+ return &SyncPruneService{snapshotDAO: snapshotDAO, deleter: deleter, store: store}
+}
+
+// ClearSnapshot removes temporary snapshot rows.
+func (s *SyncPruneService) ClearSnapshot(ctx context.Context, taskID string) error {
+ return s.snapshotDAO.Clear(ctx, taskID)
+}
+
+// AddSnapshotBatch writes one slim snapshot batch.
+func (s *SyncPruneService) AddSnapshotBatch(ctx context.Context, taskID string, documents []syncerconnector.SlimDocument) error {
+ return s.snapshotDAO.InsertBatch(ctx, taskID, documents)
+}
+
+// DeleteStale removes documents missing from the complete source snapshot.
+func (s *SyncPruneService) DeleteStale(ctx context.Context, taskContext SyncTaskContext) (int64, error) {
+ sourceIDs, err := s.snapshotDAO.ListSourceIDs(ctx, taskContext.Task.ID)
+ if err != nil {
+ return 0, err
+ }
+ sourceType := SourceType(taskContext.Connector.Source, taskContext.Connector.ID)
+ retain := RetainDocumentIDs(taskContext.Knowledgebase.ID, taskContext.Connector.ID, sourceIDs)
+ existing, err := s.store.ListIDs(ctx, taskContext.Knowledgebase.ID, sourceType)
+ if err != nil {
+ return 0, err
+ }
+ var removed int64
+ for docID := range existing {
+ if _, ok := retain[docID]; ok {
+ continue
+ }
+ if s.deleter == nil {
+ return removed, ErrSyncDocumentDeleterNotConfigured
+ }
+ if err = s.deleter.DeleteDocument(ctx, docID); err != nil {
+ return removed, err
+ }
+ removed++
+ }
+ if err = s.snapshotDAO.Clear(ctx, taskContext.Task.ID); err != nil {
+ return removed, err
+ }
+ return removed, nil
+}
+
+// RetainDocumentIDs calculates PRUNE retain IDs using both legacy and new schemes.
+func RetainDocumentIDs(kbID, connectorID string, sourceIDs []string) map[string]struct{} {
+ retain := make(map[string]struct{}, len(sourceIDs)*2)
+ for _, sourceID := range sourceIDs {
+ retain[Hash128(connectorID+":"+sourceID)] = struct{}{}
+ retain[Hash128(kbID+":"+connectorID+":"+sourceID)] = struct{}{}
+ }
+ return retain
+}
diff --git a/internal/service/sync_task_service.go b/internal/service/sync_task_service.go
new file mode 100644
index 0000000000..5d383ec1d0
--- /dev/null
+++ b/internal/service/sync_task_service.go
@@ -0,0 +1,157 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "fmt"
+ "ragflow/internal/dao"
+ "strings"
+ "time"
+)
+
+const (
+ // TaskTypeSync is the Python-compatible SYNC task type.
+ TaskTypeSync = dao.TaskTypeSync
+ // TaskTypePrune is the Python-compatible PRUNE task type.
+ TaskTypePrune = dao.TaskTypePrune
+)
+
+// SyncTaskContext contains the database rows needed to execute a task.
+type SyncTaskContext = dao.SyncTaskContext
+
+// SyncStats accumulates task-level document results.
+type SyncStats struct {
+ Added int64
+ Updated int64
+ Skipped int64
+ ErrorCount int64
+ ErrorMsg string
+}
+
+// Add merges another stats value.
+func (s *SyncStats) Add(other SyncStats) {
+ s.Added += other.Added
+ s.Updated += other.Updated
+ s.Skipped += other.Skipped
+ s.ErrorCount += other.ErrorCount
+ if other.ErrorMsg != "" {
+ if s.ErrorMsg != "" {
+ s.ErrorMsg += "\n"
+ }
+ s.ErrorMsg += other.ErrorMsg
+ }
+}
+
+// AddResult merges one document result.
+func (s *SyncStats) AddResult(result DocumentUpsertResult) {
+ switch result.Action {
+ case DocumentActionAdded:
+ s.Added++
+ case DocumentActionUpdated:
+ s.Updated++
+ case DocumentActionSkipped:
+ s.Skipped++
+ }
+}
+
+// SyncTaskService contains sync task business updates.
+type SyncTaskService struct {
+ taskDAO *dao.SyncTaskDAO
+}
+
+// NewSyncTaskService creates a task service.
+func NewSyncTaskService(taskDAO *dao.SyncTaskDAO) *SyncTaskService {
+ return &SyncTaskService{taskDAO: taskDAO}
+}
+
+// ListDueTasks returns due schedule tasks.
+func (s *SyncTaskService) ListDueTasks(ctx context.Context, now time.Time) ([]dao.SyncTask, error) {
+ tasks, err := s.taskDAO.ListDueTasks(ctx, now, 128)
+ if err != nil {
+ return nil, err
+ }
+ out := make([]dao.SyncTask, 0, len(tasks))
+ for _, task := range tasks {
+ out = append(out, dao.SyncTask{SyncLogs: task})
+ }
+ return out, nil
+}
+
+// Claim marks a scheduled task running if no other scanner claimed it first.
+func (s *SyncTaskService) Claim(ctx context.Context, taskID string) (bool, error) {
+ claimed, err := s.taskDAO.ClaimTask(ctx, taskID, time.Now().Local())
+ if err != nil || !claimed {
+ return claimed, err
+ }
+
+ taskContext, err := s.taskDAO.GetTaskContext(ctx, taskID)
+ if err != nil {
+ return true, err
+ }
+
+ return true, s.taskDAO.MarkConnectorRunning(ctx, taskContext.Connector.ID)
+}
+
+// GetContext loads a task execution context.
+func (s *SyncTaskService) GetContext(ctx context.Context, taskID string) (SyncTaskContext, error) {
+ return s.taskDAO.GetTaskContext(ctx, taskID)
+}
+
+// RescheduleClaimed puts a claimed task back into schedule state.
+func (s *SyncTaskService) RescheduleClaimed(ctx context.Context, taskID string) error {
+ return s.taskDAO.RescheduleClaimed(ctx, taskID)
+}
+
+// Fail records a failed task.
+func (s *SyncTaskService) Fail(ctx context.Context, taskID, connectorID string, err error) error {
+ message := ""
+ if err != nil {
+ message = err.Error()
+ }
+ return s.taskDAO.FailTask(ctx, taskID, connectorID, message, 1)
+}
+
+// CompleteSync commits a successful SYNC task and schedules the next one.
+func (s *SyncTaskService) CompleteSync(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, stats SyncStats) error {
+ changed := stats.Added + stats.Updated
+ return s.taskDAO.CompleteSyncTask(ctx, taskContext, pollRangeEnd, changed, changed, stats.ErrorCount, stats.ErrorMsg)
+}
+
+// CompletePrune commits a successful PRUNE task and schedules the next one.
+func (s *SyncTaskService) CompletePrune(ctx context.Context, taskContext SyncTaskContext, removed int64) error {
+ return s.taskDAO.CompletePruneTask(ctx, taskContext, removed)
+}
+
+// RecoverStaleRunning restores timed-out running tasks.
+func (s *SyncTaskService) RecoverStaleRunning(ctx context.Context, now time.Time) error {
+ _, err := s.taskDAO.RecoverStaleRunning(ctx, now)
+ return err
+}
+
+// IsFromBeginning reports whether a task is a full sync.
+func IsFromBeginning(value *string) bool {
+ if value == nil {
+ return false
+ }
+ return strings.TrimSpace(*value) == "1"
+}
+
+// SourceType returns document source_type.
+func SourceType(source, connectorID string) string {
+ return fmt.Sprintf("%s/%s", source, connectorID)
+}
diff --git a/internal/syncer/config.go b/internal/syncer/config.go
new file mode 100644
index 0000000000..4abd51b219
--- /dev/null
+++ b/internal/syncer/config.go
@@ -0,0 +1,79 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "time"
+)
+
+// Config contains runtime limits for the datasource syncer.
+type Config struct {
+ PollInterval time.Duration
+ TaskConcurrency int
+ TaskQueueSize int
+ PerTaskItemConcurrency int
+ GlobalItemConcurrency int
+ ItemRetryCount int
+ ItemRetryBaseDelay time.Duration
+}
+
+// DefaultConfig returns the first-version syncer defaults.
+func DefaultConfig() Config {
+ return Config{
+ PollInterval: 3 * time.Second,
+ TaskConcurrency: 3,
+ TaskQueueSize: 32,
+ PerTaskItemConcurrency: 4,
+ GlobalItemConcurrency: 12,
+ ItemRetryCount: 3,
+ ItemRetryBaseDelay: time.Second,
+ }
+}
+
+// Normalize fills unset or invalid config values with defaults.
+func (c Config) Normalize() Config {
+ def := DefaultConfig()
+ if c.PollInterval <= 0 {
+ c.PollInterval = def.PollInterval
+ }
+
+ if c.TaskConcurrency <= 0 {
+ c.TaskConcurrency = def.TaskConcurrency
+ }
+
+ if c.TaskQueueSize <= 0 {
+ c.TaskQueueSize = def.TaskQueueSize
+ }
+
+ if c.PerTaskItemConcurrency <= 0 {
+ c.PerTaskItemConcurrency = def.PerTaskItemConcurrency
+ }
+
+ if c.GlobalItemConcurrency <= 0 {
+ c.GlobalItemConcurrency = def.GlobalItemConcurrency
+ }
+
+ if c.ItemRetryCount <= 0 {
+ c.ItemRetryCount = def.ItemRetryCount
+ }
+
+ if c.ItemRetryBaseDelay <= 0 {
+ c.ItemRetryBaseDelay = def.ItemRetryBaseDelay
+ }
+
+ return c
+}
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.go
new file mode 100644
index 0000000000..0c0ab8bfe8
--- /dev/null
+++ b/internal/syncer/connector/github.go
@@ -0,0 +1,691 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+const (
+ defaultGitHubBatchSize = 32
+ githubItemsPerPage = 100
+ githubRequestTimeout = 60 * time.Second
+)
+
+// GitHubConnector reads GitHub issues and pull requests.
+type GitHubConnector struct {
+ owner string
+ repos []string
+ includePRs bool
+ includeIssues bool
+ token string
+ batchSize int
+ baseURL string
+ doJSON func(ctx context.Context, apiURL string, out any) (http.Header, error)
+}
+
+// NewGitHubConnector creates a GitHub connector from Python-compatible config.
+func NewGitHubConnector(config map[string]any) (*GitHubConnector, error) {
+ credentials, _ := config["credentials"].(map[string]any)
+ token, _ := credentials["github_access_token"].(string)
+ baseURL := strings.TrimRight(os.Getenv("GITHUB_CONNECTOR_BASE_URL"), "/")
+ if baseURL == "" {
+ baseURL = "https://api.github.com"
+ }
+ return &GitHubConnector{
+ owner: strings.TrimSpace(stringConfig(config["repository_owner"])),
+ repos: splitGitHubRepos(stringConfig(config["repository_name"])),
+ includePRs: configBoolDefault(config["include_pull_requests"], true),
+ includeIssues: configBoolDefault(config["include_issues"], true),
+ token: strings.TrimSpace(token),
+ batchSize: configInt(config["batch_size"], defaultGitHubBatchSize),
+ baseURL: baseURL,
+ }, nil
+}
+
+// Validate validates GitHub connector settings and credentials.
+func (c *GitHubConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("github connector is nil")
+ }
+ if c.owner == "" {
+ return fmt.Errorf("Invalid connector settings: 'repo_owner' must be provided")
+ }
+ if c.token == "" {
+ return fmt.Errorf("Missing github_access_token in credentials")
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ repos, err := c.listRepos(ctx)
+ if err != nil {
+ return err
+ }
+ if len(repos) == 0 {
+ return fmt.Errorf("found no repos for GitHub owner %s", c.owner)
+ }
+ return nil
+}
+
+// OpenSync opens one GitHub sync session.
+func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ repos, err := c.listRepos(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &githubSyncSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1, windowStart: request.WindowStart, windowEnd: request.WindowEnd}, nil
+}
+
+// OpenPrune opens one complete GitHub prune snapshot session.
+func (c *GitHubConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ repos, err := c.listRepos(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &githubPruneSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1}, nil
+}
+
+// listRepos returns configured repositories or all owner repositories.
+func (c *GitHubConnector) listRepos(ctx context.Context) ([]githubRepo, error) {
+ if len(c.repos) > 0 {
+ repos := make([]githubRepo, 0, len(c.repos))
+ for _, repoName := range c.repos {
+ var repo githubRepo
+ if _, err := c.getJSON(ctx, c.apiURL("/repos/"+url.PathEscape(c.owner)+"/"+url.PathEscape(repoName), nil), &repo); err != nil {
+ return nil, err
+ }
+ repos = append(repos, repo)
+ }
+ return repos, nil
+ }
+
+ repos, err := c.listRepoEndpoint(ctx, "/orgs/"+url.PathEscape(c.owner)+"/repos")
+ if err == nil {
+ return repos, nil
+ }
+ return c.listRepoEndpoint(ctx, "/users/"+url.PathEscape(c.owner)+"/repos")
+}
+
+// listRepoEndpoint returns all repos from an org or user endpoint.
+func (c *GitHubConnector) listRepoEndpoint(ctx context.Context, path string) ([]githubRepo, error) {
+ repos := []githubRepo{}
+ for page := 1; ; page++ {
+ query := url.Values{"per_page": {strconv.Itoa(githubItemsPerPage)}, "page": {strconv.Itoa(page)}}
+ var batch []githubRepo
+ headers, err := c.getJSON(ctx, c.apiURL(path, query), &batch)
+ if err != nil {
+ return nil, err
+ }
+ repos = append(repos, batch...)
+ if !hasNextPage(headers) || len(batch) == 0 {
+ break
+ }
+ }
+ return repos, nil
+}
+
+// listPullRequestPage returns one page of GitHub pull requests.
+func (c *GitHubConnector) listPullRequestPage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]SourceDocument, bool, error) {
+ var batch []githubPullRequest
+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/pulls", githubListQuery(page, pageSize)), &batch)
+ if err != nil {
+ return nil, false, err
+ }
+ documents := make([]SourceDocument, 0, len(batch))
+ doneByWindow := false
+ for _, pr := range batch {
+ if beforeOrAtWindowStart(pr.UpdatedAt, windowStart) {
+ doneByWindow = true
+ break
+ }
+ if afterWindowEnd(pr.UpdatedAt, windowEnd) {
+ continue
+ }
+ documents = append(documents, pr.toSourceDocument(fullName))
+ }
+ done := doneByWindow || !hasNextPage(headers) || len(batch) == 0
+ return documents, done, nil
+}
+
+// listIssuePage returns one page of GitHub issues.
+func (c *GitHubConnector) listIssuePage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]SourceDocument, bool, error) {
+ var batch []githubIssue
+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/issues", githubListQuery(page, pageSize)), &batch)
+ if err != nil {
+ return nil, false, err
+ }
+ documents := make([]SourceDocument, 0, len(batch))
+ doneByWindow := false
+ for _, issue := range batch {
+ if issue.PullRequest != nil {
+ continue
+ }
+ if beforeOrAtWindowStart(issue.UpdatedAt, windowStart) {
+ doneByWindow = true
+ break
+ }
+ if afterWindowEnd(issue.UpdatedAt, windowEnd) {
+ continue
+ }
+ documents = append(documents, issue.toSourceDocument(fullName))
+ }
+ done := doneByWindow || !hasNextPage(headers) || len(batch) == 0
+ return documents, done, nil
+}
+
+// listPullRequestSlimPage returns one page of GitHub pull request IDs.
+func (c *GitHubConnector) listPullRequestSlimPage(ctx context.Context, fullName string, page, pageSize int) ([]SlimDocument, bool, error) {
+ var batch []githubPullRequest
+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/pulls", githubListQuery(page, pageSize)), &batch)
+ if err != nil {
+ return nil, false, err
+ }
+ documents := make([]SlimDocument, 0, len(batch))
+ for _, pr := range batch {
+ documents = append(documents, SlimDocument{SourceID: pr.HTMLURL})
+ }
+ return documents, !hasNextPage(headers) || len(batch) == 0, nil
+}
+
+// listIssueSlimPage returns one page of GitHub issue IDs.
+func (c *GitHubConnector) listIssueSlimPage(ctx context.Context, fullName string, page, pageSize int) ([]SlimDocument, bool, error) {
+ var batch []githubIssue
+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/issues", githubListQuery(page, pageSize)), &batch)
+ if err != nil {
+ return nil, false, err
+ }
+ documents := make([]SlimDocument, 0, len(batch))
+ for _, issue := range batch {
+ if issue.PullRequest != nil {
+ continue
+ }
+ documents = append(documents, SlimDocument{SourceID: issue.HTMLURL})
+ }
+ return documents, !hasNextPage(headers) || len(batch) == 0, nil
+}
+
+// githubListQuery builds standard GitHub list query parameters.
+func githubListQuery(page, pageSize int) url.Values {
+ if pageSize <= 0 {
+ pageSize = defaultGitHubBatchSize
+ }
+ return url.Values{
+ "state": {"all"},
+ "sort": {"updated"},
+ "direction": {"desc"},
+ "per_page": {strconv.Itoa(pageSize)},
+ "page": {strconv.Itoa(page)},
+ }
+}
+
+// getJSON fetches a GitHub API response into out.
+func (c *GitHubConnector) getJSON(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ if c.doJSON != nil {
+ return c.doJSON(ctx, apiURL, out)
+ }
+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
+ if err != nil {
+ return nil, err
+ }
+ client := utility.PinnedHTTPClient(hostname, resolvedIP, githubRequestTimeout)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Accept", "application/vnd.github+json")
+ req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch GitHub API: %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode >= 400 {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ return nil, fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+ }
+ if err = json.NewDecoder(resp.Body).Decode(out); err != nil {
+ return nil, err
+ }
+ return resp.Header.Clone(), nil
+}
+
+// apiURL builds a GitHub API URL.
+func (c *GitHubConnector) apiURL(path string, query url.Values) string {
+ path = strings.TrimLeft(path, "/")
+ u := strings.TrimRight(c.baseURL, "/") + "/" + path
+ if len(query) == 0 {
+ return u
+ }
+ return u + "?" + query.Encode()
+}
+
+type githubSyncSession struct {
+ connector *GitHubConnector
+ repos []githubRepo
+ repoIndex int
+ stage string
+ page int
+ batchSize int
+ windowStart *time.Time
+ windowEnd time.Time
+ buffer []SourceDocument
+}
+
+// NextBatch returns the next GitHub document batch.
+func (s *githubSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ documents := make([]SourceDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ if s.repoIndex >= len(s.repos) {
+ if len(documents) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ break
+ }
+ batch, err := s.nextDocumentPage(ctx)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ remaining := s.batchSize - len(documents)
+ if len(batch) > remaining {
+ documents = append(documents, batch[:remaining]...)
+ s.buffer = append(s.buffer, batch[remaining:]...)
+ break
+ }
+ documents = append(documents, batch...)
+ }
+ return SyncBatch{Documents: documents}, nil
+}
+
+// Close closes the GitHub sync session.
+func (s *githubSyncSession) Close() error {
+ return nil
+}
+
+type githubPruneSession struct {
+ connector *GitHubConnector
+ repos []githubRepo
+ repoIndex int
+ stage string
+ page int
+ batchSize int
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next GitHub prune snapshot batch.
+func (s *githubPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ if s.repoIndex >= len(s.repos) {
+ if len(documents) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+ break
+ }
+ batch, err := s.nextSlimPage(ctx)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ remaining := s.batchSize - len(documents)
+ if len(batch) > remaining {
+ documents = append(documents, batch[:remaining]...)
+ s.buffer = append(s.buffer, batch[remaining:]...)
+ break
+ }
+ documents = append(documents, batch...)
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the GitHub prune session.
+func (s *githubPruneSession) Close() error {
+ return nil
+}
+
+const (
+ githubStagePRs = "prs"
+ githubStageIssues = "issues"
+)
+
+// nextDocumentPage fetches one GitHub API page for sync.
+func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {
+ repo := s.repos[s.repoIndex]
+ switch s.stage {
+ case githubStagePRs:
+ if !s.connector.includePRs {
+ s.advanceStage()
+ return nil, nil
+ }
+ docs, done, err := s.connector.listPullRequestPage(ctx, repo.FullName, s.page, s.batchSize, s.windowStart, s.windowEnd)
+ if err != nil {
+ return nil, err
+ }
+ if done {
+ s.advanceStage()
+ } else {
+ s.page++
+ }
+ return docs, nil
+ case githubStageIssues:
+ if !s.connector.includeIssues {
+ s.advanceRepo()
+ return nil, nil
+ }
+ docs, done, err := s.connector.listIssuePage(ctx, repo.FullName, s.page, s.batchSize, s.windowStart, s.windowEnd)
+ if err != nil {
+ return nil, err
+ }
+ if done {
+ s.advanceRepo()
+ } else {
+ s.page++
+ }
+ return docs, nil
+ default:
+ s.advanceRepo()
+ return nil, nil
+ }
+}
+
+// nextSlimPage fetches one GitHub API page for prune.
+func (s *githubPruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {
+ repo := s.repos[s.repoIndex]
+ switch s.stage {
+ case githubStagePRs:
+ if !s.connector.includePRs {
+ s.advanceStage()
+ return nil, nil
+ }
+ docs, done, err := s.connector.listPullRequestSlimPage(ctx, repo.FullName, s.page, s.batchSize)
+ if err != nil {
+ return nil, err
+ }
+ if done {
+ s.advanceStage()
+ } else {
+ s.page++
+ }
+ return docs, nil
+ case githubStageIssues:
+ if !s.connector.includeIssues {
+ s.advanceRepo()
+ return nil, nil
+ }
+ docs, done, err := s.connector.listIssueSlimPage(ctx, repo.FullName, s.page, s.batchSize)
+ if err != nil {
+ return nil, err
+ }
+ if done {
+ s.advanceRepo()
+ } else {
+ s.page++
+ }
+ return docs, nil
+ default:
+ s.advanceRepo()
+ return nil, nil
+ }
+}
+
+// advanceStage moves a GitHub session from PRs to issues.
+func (s *githubSyncSession) advanceStage() {
+ s.stage = githubStageIssues
+ s.page = 1
+}
+
+// advanceRepo moves a GitHub session to the next repository.
+func (s *githubSyncSession) advanceRepo() {
+ s.repoIndex++
+ s.stage = githubStagePRs
+ s.page = 1
+}
+
+// advanceStage moves a GitHub prune session from PRs to issues.
+func (s *githubPruneSession) advanceStage() {
+ s.stage = githubStageIssues
+ s.page = 1
+}
+
+// advanceRepo moves a GitHub prune session to the next repository.
+func (s *githubPruneSession) advanceRepo() {
+ s.repoIndex++
+ s.stage = githubStagePRs
+ s.page = 1
+}
+
+type githubRepo struct {
+ FullName string `json:"full_name"`
+}
+
+type githubPullRequest struct {
+ HTMLURL string `json:"html_url"`
+ Number int `json:"number"`
+ Title string `json:"title"`
+ Body string `json:"body"`
+ State string `json:"state"`
+ MergedAt *time.Time `json:"merged_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ CreatedAt *time.Time `json:"created_at"`
+ ClosedAt *time.Time `json:"closed_at"`
+ User *githubUser `json:"user"`
+ Assignees []githubUser `json:"assignees"`
+ Labels []githubLabel `json:"labels"`
+}
+
+// toSourceDocument converts a pull request into the syncer model.
+func (p githubPullRequest) toSourceDocument(repo string) SourceDocument {
+ body := []byte(p.Body)
+ return SourceDocument{
+ SourceID: p.HTMLURL,
+ SemanticIdentifier: fmt.Sprintf("%d:%s", p.Number, sanitizeGitHubName(p.Title, "md")),
+ Extension: ".md",
+ Blob: body,
+ UpdatedAt: p.UpdatedAt.UTC(),
+ SizeBytes: int64(len(body)),
+ Metadata: map[string]any{
+ "object_type": "PullRequest",
+ "id": strconv.Itoa(p.Number),
+ "state": p.State,
+ "repo": repo,
+ "merged": strconv.FormatBool(p.MergedAt != nil),
+ "labels": githubLabelNames(p.Labels),
+ "user": p.User.metadata(),
+ "assignees": githubUsersMetadata(p.Assignees),
+ },
+ }
+}
+
+type githubIssue struct {
+ HTMLURL string `json:"html_url"`
+ Number int `json:"number"`
+ Title string `json:"title"`
+ Body string `json:"body"`
+ State string `json:"state"`
+ UpdatedAt time.Time `json:"updated_at"`
+ CreatedAt *time.Time `json:"created_at"`
+ ClosedAt *time.Time `json:"closed_at"`
+ User *githubUser `json:"user"`
+ Assignees []githubUser `json:"assignees"`
+ Labels []githubLabel `json:"labels"`
+ PullRequest *struct{} `json:"pull_request"`
+}
+
+// toSourceDocument converts an issue into the syncer model.
+func (i githubIssue) toSourceDocument(repo string) SourceDocument {
+ body := []byte(i.Body)
+ return SourceDocument{
+ SourceID: i.HTMLURL,
+ SemanticIdentifier: fmt.Sprintf("%d:%s", i.Number, sanitizeGitHubName(i.Title, "md")),
+ Extension: ".md",
+ Blob: body,
+ UpdatedAt: i.UpdatedAt.UTC(),
+ SizeBytes: int64(len(body)),
+ Metadata: map[string]any{
+ "object_type": "Issue",
+ "id": strconv.Itoa(i.Number),
+ "state": i.State,
+ "repo": repo,
+ "labels": githubLabelNames(i.Labels),
+ "user": i.User.metadata(),
+ "assignees": githubUsersMetadata(i.Assignees),
+ },
+ }
+}
+
+type githubUser struct {
+ Login string `json:"login"`
+ Name string `json:"name"`
+ Email string `json:"email"`
+}
+
+// metadata returns Python-like user metadata.
+func (u *githubUser) metadata() map[string]string {
+ if u == nil {
+ return nil
+ }
+ out := map[string]string{}
+ if u.Login != "" {
+ out["login"] = u.Login
+ }
+ if u.Name != "" {
+ out["name"] = u.Name
+ }
+ if u.Email != "" {
+ out["email"] = u.Email
+ }
+ return out
+}
+
+type githubLabel struct {
+ Name string `json:"name"`
+}
+
+// githubLabelNames returns label names.
+func githubLabelNames(labels []githubLabel) []string {
+ out := make([]string, 0, len(labels))
+ for _, label := range labels {
+ if label.Name != "" {
+ out = append(out, label.Name)
+ }
+ }
+ return out
+}
+
+// githubUsersMetadata returns metadata for users.
+func githubUsersMetadata(users []githubUser) []map[string]string {
+ out := make([]map[string]string, 0, len(users))
+ for i := range users {
+ out = append(out, (&users[i]).metadata())
+ }
+ return out
+}
+
+// hasNextPage reports whether a GitHub Link header has rel next.
+func hasNextPage(headers http.Header) bool {
+ return strings.Contains(headers.Get("Link"), `rel="next"`)
+}
+
+// beforeOrAtWindowStart reports whether an updated time is outside the lower bound.
+func beforeOrAtWindowStart(updatedAt time.Time, windowStart *time.Time) bool {
+ return windowStart != nil && !updatedAt.After(*windowStart)
+}
+
+// afterWindowEnd reports whether an updated time is outside the upper bound.
+func afterWindowEnd(updatedAt time.Time, windowEnd time.Time) bool {
+ return !windowEnd.IsZero() && updatedAt.After(windowEnd)
+}
+
+// splitGitHubRepos parses Python's comma-separated repository_name config.
+func splitGitHubRepos(value string) []string {
+ parts := strings.Split(value, ",")
+ repos := make([]string, 0, len(parts))
+ for _, part := range parts {
+ part = strings.TrimSpace(part)
+ if part != "" {
+ repos = append(repos, part)
+ }
+ }
+ return repos
+}
+
+// stringConfig reads a string config value.
+func stringConfig(value any) string {
+ if value == nil {
+ return ""
+ }
+ return fmt.Sprint(value)
+}
+
+// configBoolDefault reads a Python JSON bool/string flag with a default.
+func configBoolDefault(value any, fallback bool) bool {
+ if value == nil {
+ return fallback
+ }
+ switch typed := value.(type) {
+ case bool:
+ return typed
+ case string:
+ switch strings.ToLower(strings.TrimSpace(typed)) {
+ case "1", "true", "yes", "on":
+ return true
+ case "0", "false", "no", "off":
+ return false
+ }
+ }
+ return fallback
+}
+
+// sanitizeGitHubName mirrors Python's sanitized markdown filename intent.
+func sanitizeGitHubName(name, extension string) string {
+ name = strings.TrimSpace(name)
+ if name == "" {
+ name = "github"
+ }
+ replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", `"`, "_", "<", "_", ">", "_", "|", "_")
+ name = replacer.Replace(name)
+ if !strings.HasSuffix(strings.ToLower(name), "."+extension) {
+ name += "." + extension
+ }
+ return name
+}
diff --git a/internal/syncer/connector/github_test.go b/internal/syncer/connector/github_test.go
new file mode 100644
index 0000000000..7ceae7b19f
--- /dev/null
+++ b/internal/syncer/connector/github_test.go
@@ -0,0 +1,101 @@
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/url"
+ "testing"
+)
+
+// TestGitHubConnectorOpenPrune verifies PRUNE returns Python-compatible html_url IDs.
+func TestGitHubConnectorOpenPrune(t *testing.T) {
+ connector, err := NewGitHubConnector(map[string]any{
+ "repository_owner": "openai",
+ "repository_name": "ragflow",
+ "include_pull_requests": true,
+ "include_issues": true,
+ "batch_size": 10,
+ "credentials": map[string]any{"github_access_token": "token"},
+ })
+ if err != nil {
+ t.Fatalf("NewGitHubConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.github.test"
+ connector.doJSON = githubFixtureDoJSON(t)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ got := []string{}
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ want := []string{
+ "https://github.com/openai/ragflow/pull/7",
+ "https://github.com/openai/ragflow/issues/3",
+ }
+ if len(got) != len(want) {
+ t.Fatalf("ids len = %d, want %d: %v", len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("ids[%d] = %q, want %q", i, got[i], want[i])
+ }
+ }
+}
+
+// githubFixtureDoJSON returns a fixture GitHub JSON transport.
+func githubFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ t.Helper()
+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse api url: %v", err)
+ }
+ fixtures := map[string]string{
+ "/repos/openai/ragflow": `{"full_name":"openai/ragflow"}`,
+ "/repos/openai/ragflow/pulls": `[{
+ "html_url":"https://github.com/openai/ragflow/pull/7",
+ "number":7,
+ "title":"Add syncer",
+ "body":"PR body",
+ "state":"open",
+ "updated_at":"2026-01-03T00:00:00Z",
+ "labels":[{"name":"sync"}],
+ "user":{"login":"alice"}
+ }]`,
+ "/repos/openai/ragflow/issues": `[{
+ "html_url":"https://github.com/openai/ragflow/issues/3",
+ "number":3,
+ "title":"Prune bug",
+ "body":"Issue body",
+ "state":"open",
+ "updated_at":"2026-01-02T00:00:00Z",
+ "labels":[{"name":"bug"}],
+ "user":{"login":"bob"}
+ },{
+ "html_url":"https://github.com/openai/ragflow/pull/7",
+ "number":7,
+ "title":"PR shadow",
+ "body":"skip me",
+ "state":"open",
+ "updated_at":"2026-01-03T00:00:00Z",
+ "pull_request":{}
+ }]`,
+ }
+ body, ok := fixtures[parsed.Path]
+ if !ok {
+ t.Fatalf("unexpected api path %s", parsed.Path)
+ }
+ if err = json.Unmarshal([]byte(body), out); err != nil {
+ t.Fatalf("decode fixture: %v", err)
+ }
+ return http.Header{}, nil
+ }
+}
diff --git a/internal/syncer/connector/interface.go b/internal/syncer/connector/interface.go
new file mode 100644
index 0000000000..c63f145f1b
--- /dev/null
+++ b/internal/syncer/connector/interface.go
@@ -0,0 +1,51 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import "context"
+
+// SyncSession streams source documents for one fixed sync window.
+type SyncSession interface {
+ // NextBatch returns the next source document batch.
+ NextBatch(ctx context.Context) (SyncBatch, error)
+ // Close releases connector resources.
+ Close() error
+}
+
+// Fetcher lazily downloads a source document body.
+type Fetcher interface {
+ // Fetch downloads a delayed source document body.
+ Fetch(ctx context.Context, ref FetchReference) ([]byte, error)
+}
+
+// PruneSession streams a complete slim source snapshot.
+type PruneSession interface {
+ // NextBatch returns the next slim snapshot batch.
+ NextBatch(ctx context.Context) (PruneBatch, error)
+ // Close releases connector resources.
+ Close() error
+}
+
+// Connector validates configuration and opens sync/prune sessions.
+type Connector interface {
+ // Validate validates connector configuration and credentials.
+ Validate(ctx context.Context) error
+ // OpenSync opens one synchronization session.
+ OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error)
+ // OpenPrune opens one complete prune snapshot session.
+ OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error)
+}
diff --git a/internal/syncer/connector/mock/mock.go b/internal/syncer/connector/mock/mock.go
new file mode 100644
index 0000000000..7a1e3c8bad
--- /dev/null
+++ b/internal/syncer/connector/mock/mock.go
@@ -0,0 +1,109 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package mock
+
+import (
+ "context"
+ "io"
+ syncerconnector "ragflow/internal/syncer/connector"
+)
+
+// Connector is a programmable connector for syncer tests.
+type Connector struct {
+ ValidateErr error
+ SyncBatches []syncerconnector.SyncBatch
+ SyncErrAt int
+ PruneBatches []syncerconnector.PruneBatch
+ PruneErrAt int
+ FetchBlobs map[string][]byte
+ OnSyncBatch func(index int)
+ OnPruneBatch func(index int)
+}
+
+// Validate returns the configured validation error.
+func (c *Connector) Validate(ctx context.Context) error {
+ return c.ValidateErr
+}
+
+// OpenSync opens a mock sync session.
+func (c *Connector) OpenSync(ctx context.Context, request syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {
+ return &SyncSession{connector: c}, nil
+}
+
+// OpenPrune opens a mock prune session.
+func (c *Connector) OpenPrune(ctx context.Context, request syncerconnector.PruneRequest) (syncerconnector.PruneSession, error) {
+ return &PruneSession{connector: c}, nil
+}
+
+// SyncSession streams configured sync batches.
+type SyncSession struct {
+ connector *Connector
+ index int
+}
+
+// NextBatch returns the next configured sync batch.
+func (s *SyncSession) NextBatch(ctx context.Context) (syncerconnector.SyncBatch, error) {
+ if s.connector.SyncErrAt > 0 && s.index+1 == s.connector.SyncErrAt {
+ return syncerconnector.SyncBatch{}, io.ErrUnexpectedEOF
+ }
+ if s.index >= len(s.connector.SyncBatches) {
+ return syncerconnector.SyncBatch{}, io.EOF
+ }
+ if s.connector.OnSyncBatch != nil {
+ s.connector.OnSyncBatch(s.index)
+ }
+ batch := s.connector.SyncBatches[s.index]
+ s.index++
+ return batch, nil
+}
+
+// Fetch returns a configured lazy blob.
+func (s *SyncSession) Fetch(ctx context.Context, ref syncerconnector.FetchReference) ([]byte, error) {
+ return s.connector.FetchBlobs[ref.Key], nil
+}
+
+// Close closes the mock sync session.
+func (s *SyncSession) Close() error {
+ return nil
+}
+
+// PruneSession streams configured prune batches.
+type PruneSession struct {
+ connector *Connector
+ index int
+}
+
+// NextBatch returns the next configured prune batch.
+func (s *PruneSession) NextBatch(ctx context.Context) (syncerconnector.PruneBatch, error) {
+ if s.connector.PruneErrAt > 0 && s.index+1 == s.connector.PruneErrAt {
+ return syncerconnector.PruneBatch{}, io.ErrUnexpectedEOF
+ }
+ if s.index >= len(s.connector.PruneBatches) {
+ return syncerconnector.PruneBatch{}, io.EOF
+ }
+ if s.connector.OnPruneBatch != nil {
+ s.connector.OnPruneBatch(s.index)
+ }
+ batch := s.connector.PruneBatches[s.index]
+ s.index++
+ return batch, nil
+}
+
+// Close closes the mock prune session.
+func (s *PruneSession) Close() error {
+ return nil
+}
diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.go
new file mode 100644
index 0000000000..4454b0f866
--- /dev/null
+++ b/internal/syncer/connector/models.go
@@ -0,0 +1,70 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import "time"
+
+// SourceDocument is the normalized document emitted by datasource connectors.
+type SourceDocument struct {
+ SourceID string
+ SemanticIdentifier string
+ Extension string
+ Blob []byte
+ FetchRef *FetchReference
+ UpdatedAt time.Time
+ SizeBytes int64
+ Metadata map[string]any
+ Fingerprint string
+}
+
+// FetchReference describes a lazy document fetch.
+type FetchReference struct {
+ Key string
+ SizeHint int64
+}
+
+// SyncRequest describes one fixed sync window.
+type SyncRequest struct {
+ TaskID string
+ ConnectorID string
+ KBID string
+ FromBeginning bool
+ WindowStart *time.Time
+ WindowEnd time.Time
+}
+
+// SyncBatch contains one serially processed batch.
+type SyncBatch struct {
+ Documents []SourceDocument
+}
+
+// PruneRequest describes one complete prune snapshot request.
+type PruneRequest struct {
+ TaskID string
+ ConnectorID string
+ KBID string
+}
+
+// SlimDocument is the minimal prune snapshot row.
+type SlimDocument struct {
+ SourceID string
+}
+
+// PruneBatch contains one slim snapshot batch.
+type PruneBatch struct {
+ Documents []SlimDocument
+}
diff --git a/internal/syncer/connector/registry.go b/internal/syncer/connector/registry.go
new file mode 100644
index 0000000000..6608fcc003
--- /dev/null
+++ b/internal/syncer/connector/registry.go
@@ -0,0 +1,76 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "fmt"
+ "sync"
+)
+
+// Factory creates a connector for a task context.
+type Factory func(ctx context.Context, taskContext any) (Connector, error)
+
+// Registry maps Python connector source names to Go connector factories.
+type Registry struct {
+ mu sync.RWMutex
+ factories map[string]Factory
+}
+
+// NewRegistry creates an empty connector registry.
+func NewRegistry() *Registry {
+ return &Registry{factories: map[string]Factory{}}
+}
+
+// Register adds or replaces a connector factory.
+func (r *Registry) Register(source string, factory Factory) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.factories[source] = factory
+}
+
+// Open creates a connector for a task context.
+func (r *Registry) Open(ctx context.Context, taskContext any) (Connector, error) {
+ row, ok := taskContext.(interface{ ConnectorSource() string })
+ if ok {
+ return r.openSource(ctx, row.ConnectorSource(), taskContext)
+ }
+ sourceProvider, ok := taskContext.(interface{ Source() string })
+ if ok {
+ return r.openSource(ctx, sourceProvider.Source(), taskContext)
+ }
+ rowContext, ok := taskContext.(struct{ Connector struct{ Source string } })
+ if ok {
+ return r.openSource(ctx, rowContext.Connector.Source, taskContext)
+ }
+ source := ""
+ if value, ok := any(taskContext).(interface{ GetSource() string }); ok {
+ source = value.GetSource()
+ }
+ return r.openSource(ctx, source, taskContext)
+}
+
+// openSource creates a connector for a known source.
+func (r *Registry) openSource(ctx context.Context, source string, taskContext any) (Connector, error) {
+ r.mu.RLock()
+ factory := r.factories[source]
+ r.mu.RUnlock()
+ if factory == nil {
+ return nil, fmt.Errorf("unsupported connector source %q", source)
+ }
+ return factory(ctx, taskContext)
+}
diff --git a/internal/syncer/connector/rss.go b/internal/syncer/connector/rss.go
new file mode 100644
index 0000000000..dc626206e0
--- /dev/null
+++ b/internal/syncer/connector/rss.go
@@ -0,0 +1,538 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "bytes"
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "encoding/xml"
+ "fmt"
+ "html"
+ "io"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "ragflow/internal/utility"
+
+ "github.com/zeebo/xxh3"
+)
+
+const (
+ defaultRSSBatchSize = 32
+ maxRSSFeedSize = 32 * 1024 * 1024
+ rssFetchTimeout = 60 * time.Second
+ rssFetchRetryCount = 3
+)
+
+var (
+ rssHTMLScriptStyleRE = regexp.MustCompile(`(?is)<(script|style)[^>]*>.*?</(script|style)>`)
+ rssHTMLTagRE = regexp.MustCompile(`(?s)<[^>]+>`)
+ rssWhitespaceRE = regexp.MustCompile(`[ \t]+`)
+ rssNewlineRE = regexp.MustCompile(`\n{3,}`)
+)
+
+// RSSConnector reads RSS and Atom feeds.
+type RSSConnector struct {
+ feedURL string
+ batchSize int
+ fetchFeed func(ctx context.Context, feedURL string) ([]byte, error)
+ entries []rssEntry
+}
+
+// NewRSSConnector creates an RSS connector from Python-compatible config.
+func NewRSSConnector(config map[string]any) (*RSSConnector, error) {
+ feedURL, _ := config["feed_url"].(string)
+ batchSize := configInt(config["batch_size"], defaultRSSBatchSize)
+ return &RSSConnector{feedURL: strings.TrimSpace(feedURL), batchSize: batchSize}, nil
+}
+
+// Validate validates RSS connector settings and feed readability.
+func (c *RSSConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("rss connector is nil")
+ }
+ if c.feedURL == "" {
+ return fmt.Errorf("RSS feed URL is required")
+ }
+
+ parsed, err := url.Parse(c.feedURL)
+ if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+ return fmt.Errorf("invalid RSS feed URL")
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return fmt.Errorf("RSS feed URL must use HTTP or HTTPS")
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+
+ entries, err := c.loadEntries(ctx)
+ if err != nil {
+ return err
+ }
+ if len(entries) == 0 {
+ return fmt.Errorf("RSS feed contains no entries")
+ }
+
+ c.entries = entries
+ return nil
+}
+
+// OpenSync opens one RSS sync session.
+func (c *RSSConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ entries, err := c.loadEntries(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ documents := make([]SourceDocument, 0, len(entries))
+ for _, entry := range entries {
+ if !request.WindowEnd.IsZero() && entry.updatedAt.After(request.WindowEnd) {
+ continue
+ }
+ if !request.FromBeginning && request.WindowStart != nil { // not reindex
+ if !entry.updatedAt.After(*request.WindowStart) {
+ continue
+ }
+ }
+ documents = append(documents, entry.toSourceDocument(c.feedURL))
+ }
+ return &rssSyncSession{documents: documents, batchSize: c.batchSize}, nil
+}
+
+// OpenPrune opens one complete RSS prune snapshot session.
+func (c *RSSConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ entries, err := c.loadEntries(ctx)
+ if err != nil {
+ return nil, err
+ }
+ documents := make([]SlimDocument, 0, len(entries))
+ for _, entry := range entries {
+ documents = append(documents, SlimDocument{SourceID: entry.sourceID()})
+ }
+ return &rssPruneSession{documents: documents, batchSize: c.batchSize}, nil
+}
+
+// loadEntries fetches and parses the configured feed.
+func (c *RSSConnector) loadEntries(ctx context.Context) ([]rssEntry, error) {
+ if c.entries != nil {
+ return c.entries, nil
+ }
+ if c.fetchFeed != nil {
+ data, err := c.fetchFeed(ctx, c.feedURL)
+ if err != nil {
+ return nil, err
+ }
+ return parseRSSFeed(data)
+ }
+ data, _, _, err := fetchRSSFeedWithRetry(ctx, c.feedURL)
+ if err != nil {
+ return nil, err
+ }
+ entries, err := parseRSSFeed(data)
+ if err != nil {
+ return nil, err
+ }
+ return entries, nil
+}
+
+// fetchRSSFeedWithRetry fetches a feed with bounded retries for transient slowness.
+func fetchRSSFeedWithRetry(ctx context.Context, feedURL string) ([]byte, http.Header, string, error) {
+ var lastErr error
+ for attempt := 1; attempt <= rssFetchRetryCount; attempt++ {
+ data, headers, finalURL, err := utility.FetchRemoteFileSafelyWithTimeout(ctx, feedURL, maxRSSFeedSize, rssFetchTimeout)
+ if err == nil {
+ return data, headers, finalURL, nil
+ }
+ lastErr = err
+ if attempt == rssFetchRetryCount {
+ break
+ }
+ select {
+ case <-ctx.Done():
+ return nil, nil, "", ctx.Err()
+ case <-time.After(time.Duration(attempt) * time.Second):
+ }
+ }
+ return nil, nil, "", lastErr
+}
+
+type rssSyncSession struct {
+ documents []SourceDocument
+ batchSize int
+ index int
+}
+
+// NextBatch returns the next RSS document batch.
+func (s *rssSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ if s.index >= len(s.documents) {
+ return SyncBatch{}, io.EOF
+ }
+ end := s.index + s.batchSize
+ if end > len(s.documents) {
+ end = len(s.documents)
+ }
+ batch := SyncBatch{Documents: s.documents[s.index:end]}
+ s.index = end
+ return batch, nil
+}
+
+// Close closes the RSS sync session.
+func (s *rssSyncSession) Close() error {
+ return nil
+}
+
+type rssPruneSession struct {
+ documents []SlimDocument
+ batchSize int
+ index int
+}
+
+// NextBatch returns the next RSS prune snapshot batch.
+func (s *rssPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ if s.index >= len(s.documents) {
+ return PruneBatch{}, io.EOF
+ }
+ end := s.index + s.batchSize
+ if end > len(s.documents) {
+ end = len(s.documents)
+ }
+ batch := PruneBatch{Documents: s.documents[s.index:end]}
+ s.index = end
+ return batch, nil
+}
+
+// Close closes the RSS prune session.
+func (s *rssPruneSession) Close() error {
+ return nil
+}
+
+type rssEntry struct {
+ id string
+ link string
+ title string
+ author string
+ categories []string
+ content string
+ summary string
+ updatedAt time.Time
+}
+
+// sourceID returns the RSS source document ID.
+func (e rssEntry) sourceID() string {
+ stable := firstNonEmpty(e.id, e.link, e.title)
+ sum := md5.Sum([]byte(stable))
+ return "rss:" + hex.EncodeToString(sum[:])
+}
+
+// semanticIdentifier returns the human-readable document name seed.
+func (e rssEntry) semanticIdentifier() string {
+ return firstNonEmpty(e.title, e.link, e.id, "rss-entry")
+}
+
+// toSourceDocument converts one feed entry into the syncer model.
+func (e rssEntry) toSourceDocument(feedURL string) SourceDocument {
+ body := e.renderText()
+ fingerprint := rssFingerprint(body)
+ metadata := map[string]any{"feed_url": feedURL}
+ if e.link != "" {
+ metadata["link"] = e.link
+ }
+ if e.author != "" {
+ metadata["author"] = e.author
+ }
+ if len(e.categories) > 0 {
+ metadata["categories"] = e.categories
+ }
+ return SourceDocument{
+ SourceID: e.sourceID(),
+ SemanticIdentifier: e.semanticIdentifier(),
+ Extension: ".txt",
+ Blob: []byte(body),
+ UpdatedAt: e.updatedAt,
+ SizeBytes: int64(len(body)),
+ Metadata: metadata,
+ Fingerprint: fingerprint,
+ }
+}
+
+// renderText builds the plain-text RSS document body.
+func (e rssEntry) renderText() string {
+ parts := []string{e.semanticIdentifier()}
+ body := htmlToText(firstNonEmpty(e.content, e.summary))
+ if body != "" {
+ parts = append(parts, body)
+ }
+ return strings.Join(parts, "\n\n")
+}
+
+type rssRoot struct {
+ XMLName xml.Name
+ Channel rssChannel `xml:"channel"`
+}
+
+type rssChannel struct {
+ Items []rssXMLItem `xml:"item"`
+}
+
+type rssXMLItem struct {
+ GUID string `xml:"guid"`
+ Title string `xml:"title"`
+ Link string `xml:"link"`
+ Author string `xml:"author"`
+ Creator string `xml:"creator"`
+ PubDate string `xml:"pubDate"`
+ Published string `xml:"published"`
+ Updated string `xml:"updated"`
+ Description string `xml:"description"`
+ Summary string `xml:"summary"`
+ Encoded string `xml:"encoded"`
+ Categories []rssCategory `xml:"category"`
+}
+
+type rssCategory struct {
+ Term string `xml:"term,attr"`
+ Label string `xml:"label,attr"`
+ Text string `xml:",chardata"`
+}
+
+type atomFeed struct {
+ Entries []atomEntry `xml:"entry"`
+}
+
+type atomEntry struct {
+ ID string `xml:"id"`
+ Title string `xml:"title"`
+ Updated string `xml:"updated"`
+ Published string `xml:"published"`
+ Summary atomText `xml:"summary"`
+ Content atomText `xml:"content"`
+ Links []atomLink `xml:"link"`
+ Author atomPerson `xml:"author"`
+ Categories []atomCategory `xml:"category"`
+}
+
+type atomText struct {
+ Text string `xml:",innerxml"`
+}
+
+type atomLink struct {
+ Href string `xml:"href,attr"`
+ Rel string `xml:"rel,attr"`
+}
+
+type atomPerson struct {
+ Name string `xml:"name"`
+}
+
+type atomCategory struct {
+ Term string `xml:"term,attr"`
+ Label string `xml:"label,attr"`
+}
+
+// parseRSSFeed parses RSS 2.0 and Atom payloads.
+func parseRSSFeed(data []byte) ([]rssEntry, error) {
+ decoder := xml.NewDecoder(bytes.NewReader(data))
+ for {
+ token, err := decoder.Token()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse RSS feed: %w", err)
+ }
+ if start, ok := token.(xml.StartElement); ok {
+ switch strings.ToLower(start.Name.Local) {
+ case "rss", "rdf":
+ return parseRSSXML(data)
+ case "feed":
+ return parseAtomXML(data)
+ default:
+ return nil, fmt.Errorf("failed to parse RSS feed")
+ }
+ }
+ }
+}
+
+// parseRSSXML parses an RSS-style feed document.
+func parseRSSXML(data []byte) ([]rssEntry, error) {
+ var root rssRoot
+ if err := xml.Unmarshal(data, &root); err != nil {
+ return nil, fmt.Errorf("failed to parse RSS feed: %w", err)
+ }
+ entries := make([]rssEntry, 0, len(root.Channel.Items))
+ for _, item := range root.Channel.Items {
+ entry := rssEntry{
+ id: strings.TrimSpace(item.GUID),
+ link: strings.TrimSpace(item.Link),
+ title: strings.TrimSpace(item.Title),
+ author: firstNonEmpty(item.Author, item.Creator),
+ categories: rssCategories(item.Categories),
+ content: firstNonEmpty(item.Encoded, item.Description),
+ summary: firstNonEmpty(item.Summary, item.Description),
+ updatedAt: parseFeedTime(firstNonEmpty(item.Updated, item.Published, item.PubDate)),
+ }
+ entries = append(entries, entry.withDefaults())
+ }
+ return entries, nil
+}
+
+// parseAtomXML parses an Atom feed document.
+func parseAtomXML(data []byte) ([]rssEntry, error) {
+ var feed atomFeed
+ if err := xml.Unmarshal(data, &feed); err != nil {
+ return nil, fmt.Errorf("failed to parse RSS feed: %w", err)
+ }
+ entries := make([]rssEntry, 0, len(feed.Entries))
+ for _, item := range feed.Entries {
+ entry := rssEntry{
+ id: strings.TrimSpace(item.ID),
+ link: atomEntryLink(item.Links),
+ title: htmlToText(item.Title),
+ author: strings.TrimSpace(item.Author.Name),
+ categories: atomCategories(item.Categories),
+ content: item.Content.Text,
+ summary: item.Summary.Text,
+ updatedAt: parseFeedTime(firstNonEmpty(item.Updated, item.Published)),
+ }
+ entries = append(entries, entry.withDefaults())
+ }
+ return entries, nil
+}
+
+// withDefaults fills missing stable fields with deterministic fallbacks.
+func (e rssEntry) withDefaults() rssEntry {
+ return e
+}
+
+// rssFingerprint returns a deterministic body fingerprint.
+func rssFingerprint(body string) string {
+ sum := xxh3.Hash128([]byte(body)).Bytes()
+ return hex.EncodeToString(sum[:])
+}
+
+// rssCategories returns RSS category labels.
+func rssCategories(categories []rssCategory) []string {
+ out := make([]string, 0, len(categories))
+ for _, category := range categories {
+ value := firstNonEmpty(category.Term, category.Label, category.Text)
+ if value != "" {
+ out = append(out, value)
+ }
+ }
+ return out
+}
+
+// atomCategories returns Atom category labels.
+func atomCategories(categories []atomCategory) []string {
+ out := make([]string, 0, len(categories))
+ for _, category := range categories {
+ value := firstNonEmpty(category.Label, category.Term)
+ if value != "" {
+ out = append(out, value)
+ }
+ }
+ return out
+}
+
+// atomEntryLink returns the preferred Atom link.
+func atomEntryLink(links []atomLink) string {
+ for _, link := range links {
+ if link.Href != "" && (link.Rel == "" || link.Rel == "alternate") {
+ return strings.TrimSpace(link.Href)
+ }
+ }
+ if len(links) > 0 {
+ return strings.TrimSpace(links[0].Href)
+ }
+ return ""
+}
+
+// htmlToText removes markup and normalizes text spacing.
+func htmlToText(value string) string {
+ value = rssHTMLScriptStyleRE.ReplaceAllString(value, "")
+ value = rssHTMLTagRE.ReplaceAllString(value, "\n")
+ value = html.UnescapeString(value)
+ value = strings.ReplaceAll(value, "\r\n", "\n")
+ value = strings.ReplaceAll(value, "\r", "\n")
+ lines := strings.Split(value, "\n")
+ for i, line := range lines {
+ lines[i] = strings.TrimSpace(rssWhitespaceRE.ReplaceAllString(line, " "))
+ }
+ value = strings.Join(lines, "\n")
+ value = rssNewlineRE.ReplaceAllString(value, "\n\n")
+ return strings.TrimSpace(value)
+}
+
+// parseFeedTime parses common RSS and Atom time formats.
+func parseFeedTime(value string) time.Time {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return time.Time{}
+ }
+ layouts := []string{
+ time.RFC1123Z,
+ time.RFC1123,
+ time.RFC3339Nano,
+ time.RFC3339,
+ "Mon, 02 Jan 2006 15:04:05 -0700",
+ "Mon, 2 Jan 2006 15:04:05 -0700",
+ "2006-01-02 15:04:05",
+ }
+ for _, layout := range layouts {
+ if parsed, err := time.Parse(layout, value); err == nil {
+ return parsed.UTC()
+ }
+ }
+ return time.Time{}
+}
+
+// firstNonEmpty returns the first non-empty trimmed string.
+func firstNonEmpty(values ...string) string {
+ for _, value := range values {
+ if strings.TrimSpace(value) != "" {
+ return strings.TrimSpace(value)
+ }
+ }
+ return ""
+}
+
+// configInt reads a positive integer-like config value.
+func configInt(value any, fallback int) int {
+ switch typed := value.(type) {
+ case int:
+ if typed > 0 {
+ return typed
+ }
+ case int64:
+ if typed > 0 {
+ return int(typed)
+ }
+ case float64:
+ if typed > 0 {
+ return int(typed)
+ }
+ case string:
+ if parsed, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil && parsed > 0 {
+ return parsed
+ }
+ }
+ return fallback
+}
diff --git a/internal/syncer/connector/rss_test.go b/internal/syncer/connector/rss_test.go
new file mode 100644
index 0000000000..bfae3c2adc
--- /dev/null
+++ b/internal/syncer/connector/rss_test.go
@@ -0,0 +1,141 @@
+package connector
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "errors"
+ "io"
+ "testing"
+ "time"
+)
+
+// TestRSSConnectorOpenSyncFullAndIncremental verifies full and windowed sync.
+func TestRSSConnectorOpenSyncFullAndIncremental(t *testing.T) {
+ connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 1})
+ if err != nil {
+ t.Fatalf("NewRSSConnector failed: %v", err)
+ }
+ connector.fetchFeed = staticRSSFeed
+
+ fullSession, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-05T00:00:00Z")})
+ if err != nil {
+ t.Fatalf("OpenSync full failed: %v", err)
+ }
+ first, err := fullSession.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch first failed: %v", err)
+ }
+ if len(first.Documents) != 1 || first.Documents[0].SourceID != expectedRSSSourceID("entry-old") {
+ t.Fatalf("unexpected first full batch: %+v", first.Documents)
+ }
+ second, err := fullSession.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch second failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != expectedRSSSourceID("entry-new") {
+ t.Fatalf("unexpected second full batch: %+v", second.Documents)
+ }
+ if _, err = fullSession.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ incrementalSession, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: mustTime(t, "2026-01-04T00:00:00Z")})
+ if err != nil {
+ t.Fatalf("OpenSync incremental failed: %v", err)
+ }
+ batch, err := incrementalSession.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch incremental failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("incremental batch len = %d, want 1", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != expectedRSSSourceID("entry-new") {
+ t.Fatalf("incremental source id = %s", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "New title" {
+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
+ }
+ if string(doc.Blob) != "New title\n\nNew body" {
+ t.Fatalf("blob = %q", string(doc.Blob))
+ }
+ if doc.Metadata["link"] != "https://example.com/new" {
+ t.Fatalf("link metadata = %v", doc.Metadata["link"])
+ }
+}
+
+// TestRSSConnectorOpenPrune verifies complete slim snapshot generation.
+func TestRSSConnectorOpenPrune(t *testing.T) {
+ connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 10})
+ if err != nil {
+ t.Fatalf("NewRSSConnector failed: %v", err)
+ }
+ connector.fetchFeed = staticRSSFeed
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 3 {
+ t.Fatalf("prune snapshot len = %d, want 3", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != expectedRSSSourceID("entry-old") ||
+ batch.Documents[1].SourceID != expectedRSSSourceID("entry-new") ||
+ batch.Documents[2].SourceID != expectedRSSSourceID("entry-future") {
+ t.Fatalf("unexpected prune ids: %+v", batch.Documents)
+ }
+}
+
+// staticRSSFeed returns a fixed RSS feed for tests.
+func staticRSSFeed(ctx context.Context, feedURL string) ([]byte, error) {
+ return []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+ <channel>
+ <title>Feed</title>
+ <item>
+ <guid>entry-old</guid>
+ <title>Old title</title>
+ <link>https://example.com/old</link>
+ <pubDate>Thu, 01 Jan 2026 00:00:00 +0000</pubDate>
+ <description><![CDATA[<p>Old body</p>]]></description>
+ </item>
+ <item>
+ <guid>entry-new</guid>
+ <title>New title</title>
+ <link>https://example.com/new</link>
+ <pubDate>Sat, 03 Jan 2026 00:00:00 +0000</pubDate>
+ <description><![CDATA[<p>New body</p>]]></description>
+ <category>news</category>
+ </item>
+ <item>
+ <guid>entry-future</guid>
+ <title>Future title</title>
+ <link>https://example.com/future</link>
+ <pubDate>Tue, 06 Jan 2026 00:00:00 +0000</pubDate>
+ <description><![CDATA[<p>Future body</p>]]></description>
+ </item>
+ </channel>
+</rss>`), nil
+}
+
+// expectedRSSSourceID returns Python-compatible RSS source ID.
+func expectedRSSSourceID(value string) string {
+ sum := md5.Sum([]byte(value))
+ return "rss:" + hex.EncodeToString(sum[:])
+}
+
+// mustTime parses a test timestamp.
+func mustTime(t *testing.T, value string) time.Time {
+ t.Helper()
+ parsed, err := time.Parse(time.RFC3339, value)
+ if err != nil {
+ t.Fatalf("parse time %q: %v", value, err)
+ }
+ return parsed
+}
diff --git a/internal/syncer/connector_lock.go b/internal/syncer/connector_lock.go
new file mode 100644
index 0000000000..4bacaffa78
--- /dev/null
+++ b/internal/syncer/connector_lock.go
@@ -0,0 +1,54 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import "sync"
+
+// ConnectorLocker serializes work for a connector.
+type ConnectorLocker interface {
+ TryLock(connectorID string) bool
+ Unlock(connectorID string)
+}
+
+// ConnectorLock is a process-local connector mutex registry.
+type ConnectorLock struct {
+ mu sync.Mutex
+ locked map[string]struct{}
+}
+
+// NewConnectorLock creates an empty process-local connector lock.
+func NewConnectorLock() *ConnectorLock {
+ return &ConnectorLock{locked: map[string]struct{}{}}
+}
+
+// TryLock attempts to acquire the connector lock without blocking.
+func (l *ConnectorLock) TryLock(connectorID string) bool {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if _, ok := l.locked[connectorID]; ok {
+ return false
+ }
+ l.locked[connectorID] = struct{}{}
+ return true
+}
+
+// Unlock releases the connector lock.
+func (l *ConnectorLock) Unlock(connectorID string) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ delete(l.locked, connectorID)
+}
diff --git a/internal/syncer/prune_runner.go b/internal/syncer/prune_runner.go
new file mode 100644
index 0000000000..c8435fc3ba
--- /dev/null
+++ b/internal/syncer/prune_runner.go
@@ -0,0 +1,80 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "errors"
+ "io"
+ "ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
+)
+
+// PruneRunner executes one PRUNE task after collecting a full slim snapshot.
+type PruneRunner struct {
+ taskService *service.SyncTaskService
+ pruneService *service.SyncPruneService
+}
+
+// NewPruneRunner creates a PRUNE runner.
+func NewPruneRunner(taskService *service.SyncTaskService, pruneService *service.SyncPruneService) *PruneRunner {
+ return &PruneRunner{taskService: taskService, pruneService: pruneService}
+}
+
+// Run collects the full prune snapshot before deleting stale documents.
+func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {
+ if r.pruneService == nil {
+ return errors.New("prune service is not configured")
+ }
+ session, err := connector.OpenPrune(ctx, syncerconnector.PruneRequest{
+ TaskID: taskContext.Task.ID,
+ ConnectorID: taskContext.Connector.ID,
+ KBID: taskContext.Knowledgebase.ID,
+ })
+ if err != nil {
+ return err
+ }
+ defer session.Close()
+
+ if err = r.pruneService.ClearSnapshot(ctx, taskContext.Task.ID); err != nil {
+ return err
+ }
+ clearSnapshot := func() {
+ _ = r.pruneService.ClearSnapshot(context.WithoutCancel(ctx), taskContext.Task.ID)
+ }
+ for {
+ batch, nextErr := session.NextBatch(ctx)
+ if errors.Is(nextErr, io.EOF) {
+ break
+ }
+ if nextErr != nil {
+ clearSnapshot()
+ return nextErr
+ }
+ if err = r.pruneService.AddSnapshotBatch(ctx, taskContext.Task.ID, batch.Documents); err != nil {
+ clearSnapshot()
+ return err
+ }
+ }
+
+ removed, err := r.pruneService.DeleteStale(ctx, taskContext)
+ if err != nil {
+ clearSnapshot()
+ return err
+ }
+ return r.taskService.CompletePrune(ctx, taskContext, removed)
+}
diff --git a/internal/syncer/recovery.go b/internal/syncer/recovery.go
new file mode 100644
index 0000000000..1fea849a1f
--- /dev/null
+++ b/internal/syncer/recovery.go
@@ -0,0 +1,28 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "ragflow/internal/service"
+ "time"
+)
+
+// RecoverStaleRunning restores timed-out running sync tasks to schedule.
+func RecoverStaleRunning(ctx context.Context, taskService *service.SyncTaskService, now time.Time) error {
+ return taskService.RecoverStaleRunning(ctx, now)
+}
diff --git a/internal/syncer/scheduler.go b/internal/syncer/scheduler.go
new file mode 100644
index 0000000000..0cc775b646
--- /dev/null
+++ b/internal/syncer/scheduler.go
@@ -0,0 +1,90 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "errors"
+ "ragflow/internal/common"
+ "ragflow/internal/service"
+ "time"
+)
+
+// TaskEnvelope is the only payload sent through the task queue.
+type TaskEnvelope struct {
+ TaskID string
+}
+
+// Scheduler scans due work and enqueues claimed task IDs.
+type Scheduler struct {
+ pollInterval time.Duration
+ queue chan<- TaskEnvelope
+ taskService *service.SyncTaskService
+}
+
+// NewScheduler creates a global scheduler for datasource sync tasks.
+func NewScheduler(pollInterval time.Duration, queue chan<- TaskEnvelope, taskService *service.SyncTaskService) *Scheduler {
+ return &Scheduler{pollInterval: pollInterval, queue: queue, taskService: taskService}
+}
+
+// Run starts recovery and periodic task discovery.
+func (s *Scheduler) Run(ctx context.Context) error {
+ if err := s.scan(ctx); err != nil && ctx.Err() == nil {
+ common.Error("syncer scheduler scan failed", err)
+ }
+ ticker := time.NewTicker(s.pollInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-ticker.C:
+ if err := s.scan(ctx); err != nil && ctx.Err() == nil {
+ common.Error("syncer scheduler scan failed", err)
+ }
+ }
+ }
+}
+
+// scan claims due tasks and places their IDs on the bounded queue.
+func (s *Scheduler) scan(ctx context.Context) error {
+ now := time.Now()
+ if err := s.taskService.RecoverStaleRunning(ctx, now); err != nil {
+ return err
+ }
+ tasks, err := s.taskService.ListDueTasks(ctx, now)
+ if err != nil {
+ return err
+ }
+ var claimErr error
+ for _, task := range tasks {
+ claimed, err := s.taskService.Claim(ctx, task.ID)
+ if err != nil {
+ claimErr = errors.Join(claimErr, err)
+ continue
+ }
+ if !claimed {
+ continue
+ }
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case s.queue <- TaskEnvelope{TaskID: task.ID}:
+ }
+ }
+ return claimErr
+}
diff --git a/internal/syncer/sync_runner.go b/internal/syncer/sync_runner.go
new file mode 100644
index 0000000000..41edd8a0a3
--- /dev/null
+++ b/internal/syncer/sync_runner.go
@@ -0,0 +1,214 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand"
+ "ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
+ "sync"
+ "time"
+)
+
+// SyncRunner executes one SYNC task with serial batches and parallel items.
+type SyncRunner struct {
+ config TaskCoordinatorConfig
+ taskService *service.SyncTaskService
+ sink service.DocumentSink
+ idResolver *service.DocumentIDResolver
+ globalItems chan struct{}
+}
+
+// NewSyncRunner creates a SYNC runner.
+func NewSyncRunner(config TaskCoordinatorConfig, taskService *service.SyncTaskService, sink service.DocumentSink, idResolver *service.DocumentIDResolver, globalItems chan struct{}) *SyncRunner {
+ return &SyncRunner{config: config, taskService: taskService, sink: sink, idResolver: idResolver, globalItems: globalItems}
+}
+
+// Run executes all sync batches and commits the final waterline.
+func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {
+ if r.sink == nil {
+ return errors.New("document sink is not configured")
+ }
+ windowEnd := time.Now().UTC()
+
+ var windowStart *time.Time
+ if !service.IsFromBeginning(taskContext.Task.FromBeginning) {
+ windowStart = taskContext.Task.PollRangeStart
+ }
+
+ session, err := connector.OpenSync(ctx, syncerconnector.SyncRequest{
+ TaskID: taskContext.Task.ID,
+ ConnectorID: taskContext.Connector.ID,
+ KBID: taskContext.Knowledgebase.ID,
+ FromBeginning: windowStart == nil,
+ WindowStart: windowStart,
+ WindowEnd: windowEnd,
+ })
+ if err != nil {
+ return err
+ }
+ defer session.Close()
+
+ sourceType := service.SourceType(taskContext.Connector.Source, taskContext.Connector.ID)
+ candidateEnd := windowStart
+ stats := service.SyncStats{}
+ for {
+ batch, nextErr := session.NextBatch(ctx)
+ if errors.Is(nextErr, io.EOF) {
+ break
+ }
+ if nextErr != nil {
+ return nextErr
+ }
+ for _, doc := range batch.Documents {
+ if candidateEnd == nil || doc.UpdatedAt.After(*candidateEnd) {
+ updatedAt := doc.UpdatedAt
+ candidateEnd = &updatedAt
+ }
+ }
+ batchStats, err := r.processBatch(ctx, taskContext, sourceType, session, batch)
+ if err != nil {
+ return err
+ }
+ stats.Add(batchStats)
+ }
+
+ if candidateEnd == nil {
+ candidateEnd = &windowEnd
+ }
+ return r.taskService.CompleteSync(ctx, taskContext, *candidateEnd, stats)
+}
+
+// processBatch processes one batch with bounded document concurrency.
+func (r *SyncRunner) processBatch(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, batch syncerconnector.SyncBatch) (service.SyncStats, error) {
+ sem := make(chan struct{}, r.config.PerTaskItemConcurrency)
+ results := make(chan service.DocumentUpsertResult, len(batch.Documents))
+ errs := make(chan error, len(batch.Documents))
+
+ var wg sync.WaitGroup
+ for _, sourceDocument := range batch.Documents {
+ sourceDocument := sourceDocument
+
+ sem <- struct{}{}
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ defer func() { <-sem }()
+ result, err := r.processDocumentWithRetry(ctx, taskContext, sourceType, session, sourceDocument)
+ if err != nil {
+ errs <- err
+ return
+ }
+ results <- result
+ }()
+ }
+
+ wg.Wait()
+ close(results)
+ close(errs)
+
+ if len(errs) > 0 {
+ return service.SyncStats{}, <-errs
+ }
+ stats := service.SyncStats{}
+
+ for result := range results {
+ stats.AddResult(result)
+ }
+ return stats, nil
+}
+
+// processDocumentWithRetry retries transient item failures.
+func (r *SyncRunner) processDocumentWithRetry(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, sourceDocument syncerconnector.SourceDocument) (service.DocumentUpsertResult, error) {
+ var lastErr error
+ for attempt := 1; attempt <= r.config.ItemRetryCount; attempt++ {
+ result, err := r.processDocument(ctx, taskContext, sourceType, session, sourceDocument)
+ if err == nil {
+ return result, nil
+ }
+
+ lastErr = err
+ if !service.IsRetryable(err) || attempt == r.config.ItemRetryCount {
+ break
+ }
+
+ shift := attempt - 1
+ if shift > 30 {
+ shift = 30
+ }
+ delay := r.config.ItemRetryBaseDelay * time.Duration(1<<shift)
+ if delay < 0 {
+ delay = time.Hour
+ }
+ if jitterMax := r.config.ItemRetryBaseDelay / 2; jitterMax > 0 {
+ delay += time.Duration(rand.Int63n(int64(jitterMax)))
+ }
+
+ select {
+ case <-ctx.Done():
+ return service.DocumentUpsertResult{}, ctx.Err()
+ case <-time.After(delay):
+ }
+ }
+ return service.DocumentUpsertResult{}, lastErr
+}
+
+// processDocument resolves IDs, skips unchanged fingerprints, fetches blobs, and upserts.
+func (r *SyncRunner) processDocument(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, sourceDocument syncerconnector.SourceDocument) (service.DocumentUpsertResult, error) {
+ if r.globalItems != nil {
+ select {
+ case <-ctx.Done():
+ return service.DocumentUpsertResult{}, ctx.Err()
+ case r.globalItems <- struct{}{}:
+ defer func() { <-r.globalItems }()
+ }
+ }
+
+ resolved, err := r.idResolver.Resolve(ctx, taskContext.Knowledgebase.ID, taskContext.Connector.ID, sourceType, sourceDocument.SourceID)
+ if err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ if sourceDocument.Fingerprint != "" && resolved.StoredFingerprint == sourceDocument.Fingerprint {
+ return service.DocumentUpsertResult{DocID: resolved.DocID, Action: service.DocumentActionSkipped}, nil
+ }
+
+ if len(sourceDocument.Blob) == 0 && sourceDocument.FetchRef != nil {
+ fetcher, ok := session.(syncerconnector.Fetcher)
+ if !ok {
+ return service.DocumentUpsertResult{}, fmt.Errorf("connector session cannot fetch %s", sourceDocument.FetchRef.Key)
+ }
+ blob, err := fetcher.Fetch(ctx, *sourceDocument.FetchRef)
+ if err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ sourceDocument.Blob = blob
+ }
+
+ return r.sink.Upsert(ctx, service.DocumentUpsertInput{
+ TaskContext: taskContext,
+ SourceType: sourceType,
+ DocumentID: resolved.DocID,
+ LegacyID: resolved.LegacyID,
+ NewID: resolved.NewID,
+ SourceDocument: sourceDocument,
+ AutoParse: taskContext.Connector2Kb.AutoParse != "0",
+ })
+}
diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go
index 6df9e539b9..23cf67f88d 100644
--- a/internal/syncer/syncer.go
+++ b/internal/syncer/syncer.go
@@ -18,127 +18,136 @@ package syncer
import (
"context"
- "fmt"
- "ragflow/internal/common"
- "ragflow/internal/dao"
- "ragflow/internal/entity"
- "ragflow/internal/utility"
+ "errors"
"sync"
"time"
- "go.uber.org/zap"
+ "ragflow/internal/dao"
+ "ragflow/internal/service"
+ documentservice "ragflow/internal/service/document"
+ syncerconnector "ragflow/internal/syncer/connector"
+ "ragflow/internal/utility"
)
-// Syncer periodically polls the sync_logs table and dispatches due
-// sync/prune tasks to a fixed-size worker pool.
+// Syncer owns the scheduler, task queue, and bounded worker pool.
type Syncer struct {
- id string
- maxConcurrency int
- pollInterval time.Duration // how often each worker queries for due tasks
+ id string
+ config Config
+ queue chan TaskEnvelope
+ scheduler *Scheduler
+ worker *TaskWorker
+ cancel context.CancelFunc
+ workerGroup sync.WaitGroup
+ stopOnce sync.Once
+ ShutdownCh chan struct{}
+}
- ctx context.Context
- cancel context.CancelFunc
+// NewSyncer creates a server-compatible syncer with default dependencies.
+func NewSyncer(maxConcurrency int, pollInterval time.Duration) *Syncer {
+ // init the config
+ config := DefaultConfig()
+ config.TaskConcurrency = maxConcurrency
+ config.PollInterval = pollInterval
- workerWg sync.WaitGroup
+ taskDAO := dao.NewSyncTaskDAO(nil)
+ registry := syncerconnector.NewRegistry()
- // ShutdownCh is closed when Stop() completes.
- ShutdownCh chan struct{}
-}
+ registerBuiltInConnectors(registry)
-// NewSyncer creates a syncer with the given concurrency and poll interval.
-func NewSyncer(maxConcurrency int, pollInterval time.Duration) *Syncer {
- ctx, cancel := context.WithCancel(context.Background())
- return &Syncer{
- id: utility.GenerateUUID(),
- maxConcurrency: maxConcurrency,
- pollInterval: pollInterval,
- ctx: ctx,
- cancel: cancel,
- ShutdownCh: make(chan struct{}),
- }
+ documentService := documentservice.NewDocumentService()
+ pruneService := service.NewSyncPruneService(dao.NewSyncPruneSnapshotDAO(taskDAO.DB()), documentService, nil)
+
+ return New(config, taskDAO, registry, documentService, pruneService)
}
-// Start launches maxConcurrency worker goroutines.
-func (s *Syncer) Start() error {
- common.Info(fmt.Sprintf("Syncer %s starting with %d workers (poll every %v)",
- s.id, s.maxConcurrency, s.pollInterval))
+// New creates a datasource syncer from explicit dependencies.
+func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService) *Syncer {
+ config = config.Normalize()
+ queue := make(chan TaskEnvelope, config.TaskQueueSize)
+ locker := NewConnectorLock()
- for i := 0; i < s.maxConcurrency; i++ {
- s.workerWg.Add(1)
- go s.workerLoop(i)
- }
- return nil
-}
+ globalItems := make(chan struct{}, config.GlobalItemConcurrency)
+ taskService := service.NewSyncTaskService(taskDAO)
+ idResolver := service.NewDocumentIDResolver(service.NewGormDocumentStore())
-// Stop cancels all workers and waits for them to finish.
-func (s *Syncer) Stop() {
- common.Info(fmt.Sprintf("Stopping syncer %s", s.id))
- s.cancel()
- s.workerWg.Wait()
- close(s.ShutdownCh)
- common.Info(fmt.Sprintf("Syncer %s stopped", s.id))
+ coordinator := NewTaskCoordinator(TaskCoordinatorConfig{
+ PerTaskItemConcurrency: config.PerTaskItemConcurrency,
+ ItemRetryCount: config.ItemRetryCount,
+ ItemRetryBaseDelay: config.ItemRetryBaseDelay,
+ }, taskService, registry, sink, pruneService, idResolver, globalItems)
+
+ return &Syncer{
+ id: utility.GenerateUUID(),
+ config: config,
+ queue: queue,
+ scheduler: NewScheduler(config.PollInterval, queue, taskService),
+ worker: NewTaskWorker(queue, taskService, coordinator, locker),
+ ShutdownCh: make(chan struct{}),
+ }
}
+// ID returns this syncer process ID.
func (s *Syncer) ID() string {
+ if s == nil {
+ return ""
+ }
return s.id
}
-// workerLoop periodically polls the DB for due tasks until ctx is cancelled.
-func (s *Syncer) workerLoop(workerID int) {
- defer s.workerWg.Done()
- common.Debug(fmt.Sprintf("Syncer worker %d started", workerID))
-
- ticker := time.NewTicker(s.pollInterval)
- defer ticker.Stop()
-
- for {
- select {
- case <-s.ctx.Done():
- common.Debug(fmt.Sprintf("Syncer worker %d exiting (ctx cancelled)", workerID))
- return
- case <-ticker.C:
- s.pollAndExecute(workerID)
- }
- }
+// Start launches the scheduler and task workers with a background context.
+func (s *Syncer) Start() error {
+ return s.StartContext(context.Background())
}
-// pollAndExecute queries due sync & prune tasks, picks one, and runs it.
-func (s *Syncer) pollAndExecute(workerID int) {
- common.Info(fmt.Sprintf("Syncer worker %d polling for due tasks", workerID))
-}
+// StartContext launches the scheduler and task workers.
+func (s *Syncer) StartContext(ctx context.Context) error {
+ if s == nil {
+ return errors.New("syncer is nil")
+ }
-// executeSyncTask runs a sync task.
-func (s *Syncer) executeSyncTask(task *entity.SyncLogs) {
- common.Info("Executing sync task",
- zap.String("task_id", task.ID),
- zap.String("connector_id", task.ConnectorID),
- zap.String("kb_id", task.KbID))
- // TODO: implement actual data-source-specific sync logic.
- // For now, mark done.
- s.markTaskDone(task.ID, task.ConnectorID)
+ runCtx, cancel := context.WithCancel(ctx)
+ s.cancel = cancel
+ s.workerGroup.Add(2)
+
+ go func() {
+ defer s.workerGroup.Done()
+ _ = s.scheduler.Run(runCtx)
+ }()
+ go func() {
+ defer s.workerGroup.Done()
+ s.worker.Run(runCtx, s.config.TaskConcurrency)
+ }()
+ return nil
}
-// executePruneTask runs a prune (delete stale docs) task.
-func (s *Syncer) executePruneTask(task *entity.SyncLogs) {
- common.Info("Executing prune task",
- zap.String("task_id", task.ID),
- zap.String("connector_id", task.ConnectorID),
- zap.String("kb_id", task.KbID))
- // TODO: implement actual prune logic.
- s.markTaskDone(task.ID, task.ConnectorID)
+// Stop cancels the scheduler and waits for workers to exit.
+func (s *Syncer) Stop() {
+ if s == nil {
+ return
+ }
+ s.stopOnce.Do(func() {
+ if s.cancel != nil {
+ s.cancel()
+ }
+ s.workerGroup.Wait()
+ close(s.ShutdownCh)
+ })
}
-// markTaskDone updates task and connector status to DONE.
-func (s *Syncer) markTaskDone(taskID, connectorID string) {
- db := dao.GetDB()
- now := time.Now().Local()
-
- db.Model(&entity.SyncLogs{}).Where("id = ?", taskID).Updates(map[string]interface{}{
- "status": string(entity.TaskStatusDone),
- "update_time": now,
+// registerBuiltInConnectors registers datasource connectors available in the server binary.
+func registerBuiltInConnectors(registry *syncerconnector.Registry) {
+ registry.Register("rss", func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {
+ row, ok := taskContext.(dao.SyncTaskContext)
+ if !ok {
+ return nil, errors.New("rss connector received an invalid task context")
+ }
+ return syncerconnector.NewRSSConnector(map[string]any(row.Connector.Config))
})
- db.Model(&entity.Connector{}).Where("id = ?", connectorID).Updates(map[string]interface{}{
- "status": string(entity.TaskStatusDone),
- "update_time": now,
+ registry.Register("github", func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {
+ row, ok := taskContext.(dao.SyncTaskContext)
+ if !ok {
+ return nil, errors.New("github connector received an invalid task context")
+ }
+ return syncerconnector.NewGitHubConnector(map[string]any(row.Connector.Config))
})
}
diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go
new file mode 100644
index 0000000000..9c24e43d31
--- /dev/null
+++ b/internal/syncer/syncer_test.go
@@ -0,0 +1,502 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "errors"
+ "io"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
+ connectormock "ragflow/internal/syncer/connector/mock"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+ "gorm.io/gorm/logger"
+)
+
+// fakeSink records document writes for coordinator tests.
+type fakeSink struct {
+ mu sync.Mutex
+ delay time.Duration
+ current int
+ maxConcurrent int
+ calls []service.DocumentUpsertInput
+ errBySourceID map[string]error
+ autoParseByDoc map[string]bool
+}
+
+// Upsert records one document write.
+func (s *fakeSink) Upsert(ctx context.Context, input service.DocumentUpsertInput) (service.DocumentUpsertResult, error) {
+ s.mu.Lock()
+ s.current++
+ if s.current > s.maxConcurrent {
+ s.maxConcurrent = s.current
+ }
+ s.calls = append(s.calls, input)
+ if s.autoParseByDoc == nil {
+ s.autoParseByDoc = map[string]bool{}
+ }
+ s.autoParseByDoc[input.SourceDocument.SourceID] = input.AutoParse
+ s.mu.Unlock()
+ if s.delay > 0 {
+ select {
+ case <-ctx.Done():
+ return service.DocumentUpsertResult{}, ctx.Err()
+ case <-time.After(s.delay):
+ }
+ }
+ s.mu.Lock()
+ s.current--
+ s.mu.Unlock()
+ if err := s.errBySourceID[input.SourceDocument.SourceID]; err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ return service.DocumentUpsertResult{DocID: input.DocumentID, Action: service.DocumentActionAdded}, nil
+}
+
+// callCount returns the number of sink calls.
+func (s *fakeSink) callCount() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return len(s.calls)
+}
+
+// fakeStore provides document IDs and fingerprints.
+type fakeStore struct {
+ ids map[string]struct{}
+ fingerprints map[string]string
+}
+
+// ListIDs returns configured IDs.
+func (s fakeStore) ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error) {
+ if s.ids == nil {
+ return map[string]struct{}{}, nil
+ }
+ return s.ids, nil
+}
+
+// GetFingerprintsByIDs returns configured fingerprints for requested IDs.
+func (s fakeStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {
+ result := make(map[string]string, len(ids))
+ if s.fingerprints == nil {
+ return result, nil
+ }
+ for _, id := range ids {
+ if fingerprint, ok := s.fingerprints[id]; ok {
+ result[id] = fingerprint
+ }
+ }
+ return result, nil
+}
+
+// fakeDeleter records document deletes.
+type fakeDeleter struct {
+ mu sync.Mutex
+ deleted []string
+}
+
+// DeleteDocument records one delete.
+func (d *fakeDeleter) DeleteDocument(ctx context.Context, docID string) error {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ d.deleted = append(d.deleted, docID)
+ return nil
+}
+
+// setupSyncerDB creates a SQLite test database.
+func setupSyncerDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err = db.AutoMigrate(&entity.Connector{}, &entity.Connector2Kb{}, &entity.Knowledgebase{}, &entity.SyncLogs{}, &entity.Document{}, &dao.SyncPruneSnapshot{}); err != nil {
+ t.Fatalf("migrate sqlite: %v", err)
+ }
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+ return db
+}
+
+// insertTaskContext inserts one connector, mapping, KB, and sync task.
+func insertTaskContext(t *testing.T, db *gorm.DB, connectorID, kbID, taskID, taskType string) {
+ t.Helper()
+ now := time.Now().Add(-time.Hour).Truncate(time.Second)
+ ts := now.UnixMilli()
+ fromBeginning := "0"
+ var existingConnector int64
+ if err := db.Model(&entity.Connector{}).Where("id = ?", connectorID).Count(&existingConnector).Error; err != nil {
+ t.Fatalf("count connector: %v", err)
+ }
+ if existingConnector == 0 {
+ if err := db.Create(&entity.Connector{
+ ID: connectorID,
+ TenantID: "tenant-1",
+ Name: connectorID,
+ Source: "mock",
+ InputType: "poll",
+ Config: entity.JSONMap{"sync_deleted_files": true},
+ RefreshFreq: 0,
+ PruneFreq: 0,
+ TimeoutSecs: 1,
+ Status: dao.SyncStatusSchedule,
+ BaseModel: entity.BaseModel{UpdateDate: &now, UpdateTime: &ts},
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+ }
+ if err := db.Create(&entity.Knowledgebase{
+ ID: kbID,
+ TenantID: "tenant-1",
+ Name: kbID,
+ EmbdID: "embd",
+ CreatedBy: "tenant-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ }).Error; err != nil {
+ t.Fatalf("insert kb: %v", err)
+ }
+ if err := db.Create(&entity.Connector2Kb{ID: connectorID + kbID, ConnectorID: connectorID, KbID: kbID, AutoParse: "1"}).Error; err != nil {
+ t.Fatalf("insert mapping: %v", err)
+ }
+ if err := db.Create(&entity.SyncLogs{
+ ID: taskID,
+ ConnectorID: connectorID,
+ KbID: kbID,
+ TaskType: taskType,
+ Status: dao.SyncStatusSchedule,
+ FromBeginning: &fromBeginning,
+ TimeStarted: &now,
+ ErrorMsg: "",
+ BaseModel: entity.BaseModel{UpdateDate: &now, UpdateTime: &ts},
+ }).Error; err != nil {
+ t.Fatalf("insert task: %v", err)
+ }
+}
+
+// newTestRegistry creates a mock connector registry.
+func newTestRegistry(connectors map[string]*connectormock.Connector) *syncerconnector.Registry {
+ registry := syncerconnector.NewRegistry()
+ registry.Register("mock", func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {
+ row := taskContext.(dao.SyncTaskContext)
+ return connectors[row.Connector.ID], nil
+ })
+ return registry
+}
+
+// newCoordinator creates a test coordinator.
+func newCoordinator(taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore, perTask int) *TaskCoordinator {
+ return NewTaskCoordinator(TaskCoordinatorConfig{PerTaskItemConcurrency: perTask, ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), make(chan struct{}, 16))
+}
+
+// TestSchedulerClaimsDueTasks verifies conditional task claiming.
+func TestSchedulerClaimsDueTasks(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ queue := make(chan TaskEnvelope, 1)
+ scheduler := NewScheduler(time.Hour, queue, taskService)
+ if err := scheduler.scan(t.Context()); err != nil {
+ t.Fatalf("scan: %v", err)
+ }
+ envelope := <-queue
+ if envelope.TaskID != "task-1" {
+ t.Fatalf("TaskID = %s, want task-1", envelope.TaskID)
+ }
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusRunning {
+ t.Fatalf("status = %s, want running", task.Status)
+ }
+}
+
+// TestWorkersRunDifferentConnectorsInParallel verifies task-level parallelism.
+func TestWorkersRunDifferentConnectorsInParallel(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ insertTaskContext(t, db, "conn-2", "kb-2", "task-2", dao.TaskTypeSync)
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ now := time.Now()
+ for _, id := range []string{"task-1", "task-2"} {
+ claimed, err := taskDAO.ClaimTask(t.Context(), id, now)
+ if err != nil || !claimed {
+ t.Fatalf("claim %s: %v %v", id, claimed, err)
+ }
+ }
+ sink := &fakeSink{delay: 120 * time.Millisecond}
+ connectors := map[string]*connectormock.Connector{
+ "conn-1": {SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: now}}}}},
+ "conn-2": {SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "b", UpdatedAt: now}}}}},
+ }
+ queue := make(chan TaskEnvelope, 2)
+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(connectors), sink, nil, fakeStore{}, 1), NewConnectorLock())
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ go worker.Run(ctx, 2)
+ queue <- TaskEnvelope{TaskID: "task-1"}
+ queue <- TaskEnvelope{TaskID: "task-2"}
+ time.Sleep(40 * time.Millisecond)
+ sink.mu.Lock()
+ maxConcurrent := sink.maxConcurrent
+ sink.mu.Unlock()
+ if maxConcurrent < 2 {
+ t.Fatalf("max concurrent sink calls = %d, want >= 2", maxConcurrent)
+ }
+}
+
+// TestConnectorLockSerializesSameConnector verifies connector-level mutual exclusion.
+func TestConnectorLockSerializesSameConnector(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ insertTaskContext(t, db, "conn-1", "kb-2", "task-2", dao.TaskTypeSync)
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ for _, id := range []string{"task-1", "task-2"} {
+ if _, err := taskDAO.ClaimTask(t.Context(), id, time.Now()); err != nil {
+ t.Fatalf("claim %s: %v", id, err)
+ }
+ }
+ sink := &fakeSink{delay: 100 * time.Millisecond}
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: time.Now()}}}}}
+ queue := make(chan TaskEnvelope, 2)
+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 1), NewConnectorLock())
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ go worker.Run(ctx, 2)
+ queue <- TaskEnvelope{TaskID: "task-1"}
+ queue <- TaskEnvelope{TaskID: "task-2"}
+ time.Sleep(180 * time.Millisecond)
+ var scheduled int64
+ if err := db.Model(&entity.SyncLogs{}).Where("connector_id = ? AND status = ?", "conn-1", dao.SyncStatusSchedule).Count(&scheduled).Error; err != nil {
+ t.Fatalf("count scheduled: %v", err)
+ }
+ if scheduled == 0 {
+ t.Fatalf("expected one same-connector task to be rescheduled")
+ }
+ sink.mu.Lock()
+ maxConcurrent := sink.maxConcurrent
+ sink.mu.Unlock()
+ if maxConcurrent > 1 {
+ t.Fatalf("same connector ran concurrently: %d", maxConcurrent)
+ }
+}
+
+// TestSyncRunnerReadsBatchesSerially verifies the next batch waits for current items.
+func TestSyncRunnerReadsBatchesSerially(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ start := time.Now()
+ var secondBatchAt time.Time
+ connector := &connectormock.Connector{
+ SyncBatches: []syncerconnector.SyncBatch{
+ {Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: start}, {SourceID: "b", UpdatedAt: start}}},
+ {Documents: []syncerconnector.SourceDocument{{SourceID: "c", UpdatedAt: start}}},
+ },
+ OnSyncBatch: func(index int) {
+ if index == 1 {
+ secondBatchAt = time.Now()
+ }
+ },
+ }
+ sink := &fakeSink{delay: 80 * time.Millisecond}
+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 2)
+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")
+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {
+ t.Fatalf("execute: %v", err)
+ }
+ if secondBatchAt.Sub(start) < 70*time.Millisecond {
+ t.Fatalf("second batch was read before first batch completed")
+ }
+}
+
+// TestSyncRunnerProcessesBatchItemsInParallel verifies per-batch item concurrency.
+func TestSyncRunnerProcessesBatchItemsInParallel(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ now := time.Now()
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: now}, {SourceID: "b", UpdatedAt: now}, {SourceID: "c", UpdatedAt: now}}}}}
+ sink := &fakeSink{delay: 80 * time.Millisecond}
+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 3)
+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")
+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {
+ t.Fatalf("execute: %v", err)
+ }
+ if sink.maxConcurrent < 2 {
+ t.Fatalf("max concurrent items = %d, want >= 2", sink.maxConcurrent)
+ }
+}
+
+// TestHash128MatchesPythonGolden verifies Python-compatible xxhash128.
+func TestHash128MatchesPythonGolden(t *testing.T) {
+ cases := map[string]string{
+ "connector-1:source-1": "0b66c92e6cefff918067e3c42606cda8",
+ "kb-1:connector-1:source-1": "ef785a8871b90d910f42cf31a8155476",
+ "": "99aa06d3014798d86001c324468d497f",
+ "hello": "b5e9c1ad071b3e7fc779cfaa5e523818",
+ }
+ for input, want := range cases {
+ if got := service.Hash128(input); got != want {
+ t.Fatalf("Hash128(%q) = %s, want %s", input, got, want)
+ }
+ }
+}
+
+// TestFingerprintSkipsUnchangedDocument verifies unchanged docs are not fetched or upserted.
+func TestFingerprintSkipsUnchangedDocument(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ legacyID := service.Hash128("conn-1:source-1")
+ store := fakeStore{ids: map[string]struct{}{legacyID: {}}, fingerprints: map[string]string{legacyID: "fp-1"}}
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Fingerprint: "fp-1", FetchRef: &syncerconnector.FetchReference{Key: "lazy"}, UpdatedAt: time.Now()}}}}}
+ sink := &fakeSink{}
+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, store, 1)
+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")
+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {
+ t.Fatalf("execute: %v", err)
+ }
+ if sink.callCount() != 0 {
+ t.Fatalf("sink calls = %d, want 0", sink.callCount())
+ }
+}
+
+// TestAutoParseFlagFlowsToSink verifies connector2kb.auto_parse is preserved.
+func TestAutoParseFlagFlowsToSink(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ if err := db.Model(&entity.Connector2Kb{}).Where("connector_id = ? AND kb_id = ?", "conn-1", "kb-1").Update("auto_parse", "0").Error; err != nil {
+ t.Fatalf("disable auto_parse: %v", err)
+ }
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}
+ sink := &fakeSink{}
+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 1)
+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")
+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {
+ t.Fatalf("execute: %v", err)
+ }
+ if sink.autoParseByDoc["source-1"] {
+ t.Fatalf("auto_parse flowed as true, want false")
+ }
+}
+
+// TestBatchFailureDoesNotAdvanceWaterline verifies failed tasks keep poll_range_end.
+func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}
+ sink := &fakeSink{errBySourceID: map[string]error{"bad": errors.New("boom")}}
+ queue := make(chan TaskEnvelope, 1)
+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 1), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusFail {
+ t.Fatalf("status = %s, want fail", task.Status)
+ }
+ if task.PollRangeEnd != nil {
+ t.Fatalf("poll_range_end advanced on failure")
+ }
+}
+
+// TestRecoverStaleRunningTasks verifies timeout recovery.
+func TestRecoverStaleRunningTasks(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ started := time.Now().Add(-2 * time.Hour)
+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "time_started": started}).Error; err != nil {
+ t.Fatalf("mark running: %v", err)
+ }
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ if err := taskService.RecoverStaleRunning(t.Context(), time.Now()); err != nil {
+ t.Fatalf("recover: %v", err)
+ }
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusSchedule {
+ t.Fatalf("status = %s, want schedule", task.Status)
+ }
+}
+
+// TestPruneSnapshotFailureDoesNotDelete verifies incomplete snapshots never delete.
+func TestPruneSnapshotFailureDoesNotDelete(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypePrune)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))
+ deleter := &fakeDeleter{}
+ pruneService := service.NewSyncPruneService(dao.NewSyncPruneSnapshotDAO(db), deleter, fakeStore{ids: map[string]struct{}{"stale": {}}})
+ connector := &connectormock.Connector{PruneErrAt: 1, PruneBatches: []syncerconnector.PruneBatch{{Documents: []syncerconnector.SlimDocument{{SourceID: "keep"}}}}}
+ queue := make(chan TaskEnvelope, 1)
+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, pruneService, fakeStore{}, 1), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+ if len(deleter.deleted) != 0 {
+ t.Fatalf("deleted %v despite incomplete snapshot", deleter.deleted)
+ }
+ var count int64
+ if err := db.Model(&dao.SyncPruneSnapshot{}).Where("task_id = ?", "task-1").Count(&count).Error; err != nil {
+ t.Fatalf("count snapshot: %v", err)
+ }
+ if count != 0 {
+ t.Fatalf("snapshot rows = %d, want cleared", count)
+ }
+}
+
+// TestPruneRetainIDsIncludeLegacyAndNew verifies PRUNE retains both ID schemes.
+func TestPruneRetainIDsIncludeLegacyAndNew(t *testing.T) {
+ retain := service.RetainDocumentIDs("kb-1", "conn-1", []string{"source-1"})
+ legacy := service.Hash128("conn-1:source-1")
+ next := service.Hash128("kb-1:conn-1:source-1")
+ if _, ok := retain[legacy]; !ok {
+ t.Fatalf("legacy ID missing from retain set")
+ }
+ if _, ok := retain[next]; !ok {
+ t.Fatalf("new ID missing from retain set")
+ }
+}
+
+// TestMockSessionEOF documents the mock session EOF contract.
+func TestMockSessionEOF(t *testing.T) {
+ session, _ := (&connectormock.Connector{}).OpenSync(t.Context(), syncerconnector.SyncRequest{})
+ _, err := session.NextBatch(t.Context())
+ if !errors.Is(err, io.EOF) {
+ t.Fatalf("err = %v, want io.EOF", err)
+ }
+}
diff --git a/internal/syncer/task_coordinator.go b/internal/syncer/task_coordinator.go
new file mode 100644
index 0000000000..f129ea5da2
--- /dev/null
+++ b/internal/syncer/task_coordinator.go
@@ -0,0 +1,87 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "fmt"
+ "ragflow/internal/service"
+ syncerconnector "ragflow/internal/syncer/connector"
+ "time"
+)
+
+// ConnectorRegistry opens registered connectors by source.
+type ConnectorRegistry interface {
+ // Open creates a connector for a task context.
+ Open(ctx context.Context, taskContext any) (syncerconnector.Connector, error)
+}
+
+// TaskCoordinatorConfig controls per-task document processing.
+type TaskCoordinatorConfig struct {
+ PerTaskItemConcurrency int
+ ItemRetryCount int
+ ItemRetryBaseDelay time.Duration
+}
+
+// TaskCoordinator owns one task execution window.
+type TaskCoordinator struct {
+ config TaskCoordinatorConfig
+ taskService *service.SyncTaskService
+ registry ConnectorRegistry
+ sink service.DocumentSink
+ pruneService *service.SyncPruneService
+ idResolver *service.DocumentIDResolver
+ globalItems chan struct{}
+}
+
+// NewTaskCoordinator creates a coordinator for one claimed task at a time.
+func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTaskService, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService, idResolver *service.DocumentIDResolver, globalItems chan struct{}) *TaskCoordinator {
+ if config.PerTaskItemConcurrency <= 0 {
+ config.PerTaskItemConcurrency = 1
+ }
+
+ if config.ItemRetryCount <= 0 {
+ config.ItemRetryCount = 1
+ }
+
+ if config.ItemRetryBaseDelay <= 0 {
+ config.ItemRetryBaseDelay = time.Second
+ }
+
+ return &TaskCoordinator{config: config, taskService: taskService, registry: registry, sink: sink, pruneService: pruneService, idResolver: idResolver, globalItems: globalItems}
+}
+
+// Execute dispatches a sync_logs task by task type.
+func (c *TaskCoordinator) Execute(ctx context.Context, taskContext service.SyncTaskContext) error {
+ connector, err := c.registry.Open(ctx, taskContext)
+ if err != nil {
+ return err
+ }
+ if err = connector.Validate(ctx); err != nil {
+ return err
+ }
+ switch taskContext.Task.TaskType {
+ case service.TaskTypeSync:
+ runner := NewSyncRunner(c.config, c.taskService, c.sink, c.idResolver, c.globalItems)
+ return runner.Run(ctx, taskContext, connector)
+ case service.TaskTypePrune:
+ runner := NewPruneRunner(c.taskService, c.pruneService)
+ return runner.Run(ctx, taskContext, connector)
+ default:
+ return fmt.Errorf("unsupported sync task type %q", taskContext.Task.TaskType)
+ }
+}
diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.go
new file mode 100644
index 0000000000..230ee9bbd3
--- /dev/null
+++ b/internal/syncer/task_worker.go
@@ -0,0 +1,89 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package syncer
+
+import (
+ "context"
+ "fmt"
+ "ragflow/internal/service"
+ "sync"
+)
+
+// TaskWorker consumes claimed task envelopes and runs coordinators.
+type TaskWorker struct {
+ queue <-chan TaskEnvelope
+ taskService *service.SyncTaskService
+ coordinator *TaskCoordinator
+ locker ConnectorLocker
+}
+
+// NewTaskWorker creates a bounded task worker pool.
+func NewTaskWorker(queue <-chan TaskEnvelope, taskService *service.SyncTaskService, coordinator *TaskCoordinator, locker ConnectorLocker) *TaskWorker {
+ return &TaskWorker{queue: queue, taskService: taskService, coordinator: coordinator, locker: locker}
+}
+
+// Run starts worker goroutines and blocks until context cancellation.
+func (w *TaskWorker) Run(ctx context.Context, concurrency int) {
+ var wg sync.WaitGroup
+ for i := 0; i < concurrency; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ w.loop(ctx)
+ }()
+ }
+ <-ctx.Done()
+ wg.Wait()
+}
+
+// loop handles queue messages until cancellation.
+func (w *TaskWorker) loop(ctx context.Context) {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case envelope := <-w.queue:
+ w.handle(ctx, envelope)
+ }
+ }
+}
+
+// handle loads a claimed task and executes it under the connector lock.
+func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {
+ taskContext, err := w.taskService.GetContext(ctx, envelope.TaskID)
+ if err != nil {
+ if ctx.Err() != nil {
+ _ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID)
+ return
+ }
+ _ = w.taskService.Fail(ctx, envelope.TaskID, "", err)
+ return
+ }
+ if !w.locker.TryLock(taskContext.Connector.ID) {
+ _ = w.taskService.RescheduleClaimed(ctx, taskContext.Task.ID)
+ return
+ }
+ defer w.locker.Unlock(taskContext.Connector.ID)
+
+ if err = w.coordinator.Execute(ctx, taskContext); err != nil {
+ if ctx.Err() != nil {
+ _ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID)
+ return
+ }
+ _ = w.taskService.Fail(ctx, taskContext.Task.ID, taskContext.Connector.ID, fmt.Errorf("sync task failed: %w", err))
+ }
+}
diff --git a/internal/utility/upload.go b/internal/utility/upload.go
index 030ef1c4e6..585dc668b9 100644
--- a/internal/utility/upload.go
+++ b/internal/utility/upload.go
@@ -39,19 +39,31 @@ var (
// timeouts, and a hard size cap that rejects (rather than truncates) oversized
// bodies.
func FetchRemoteFileSafely(ctx context.Context, rawURL string, maxSize int64) ([]byte, http.Header, string, error) {
+ return FetchRemoteFileSafelyWithTimeout(ctx, rawURL, maxSize, 10*time.Second)
+}
+
+// FetchRemoteFileSafelyWithTimeout downloads rawURL with a caller-selected timeout.
+func FetchRemoteFileSafelyWithTimeout(ctx context.Context, rawURL string, maxSize int64, timeout time.Duration) ([]byte, http.Header, string, error) {
+ if timeout <= 0 {
+ timeout = 10 * time.Second
+ }
currentURL := rawURL
for redirects := 0; redirects < 10; redirects++ {
hostname, resolvedIP, err := AssertURLSafe(currentURL)
if err != nil {
return nil, nil, "", err
}
- client := PinnedHTTPClient(hostname, resolvedIP, 10*time.Second)
+ client := PinnedHTTPClient(hostname, resolvedIP, timeout)
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
// codeql[go/request-forgery] False positive: the loop above
- resp, err := client.Get(currentURL) // #nosec G107
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil)
+ if err != nil {
+ return nil, nil, "", fmt.Errorf("failed to create request: %w", err)
+ }
+ resp, err := client.Do(req) // `#nosec` G107
if err != nil {
return nil, nil, "", fmt.Errorf("failed to fetch URL: %w", err)
}