RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat[Go]: complete the base for data Syncer (#17890)
Haruko386f532f27f1f36
diff --git a/internal/dao/sync_prune_snapshot.go b/internal/dao/sync_prune_snapshot.gonew file mode 100644index 0000000000..bf4d4a964f--- /dev/null+++ b/internal/dao/sync_prune_snapshot.go@@ -0,0 +1,69 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ syncerconnector "ragflow/internal/syncer/connector"++ "gorm.io/gorm"+)++// SyncPruneSnapshot stores one source ID from a complete prune snapshot.+type SyncPruneSnapshot struct {+ TaskID string `gorm:"column:task_id;primaryKey;size:32"`+ SourceDocID string `gorm:"column:source_doc_id;primaryKey;size:512"`+}++// TableName returns the prune snapshot table name.+func (SyncPruneSnapshot) TableName() string {+ return "sync_prune_snapshot"+}++// SyncPruneSnapshotDAO manages temporary prune snapshots.+type SyncPruneSnapshotDAO struct {+ db *gorm.DB+}++// NewSyncPruneSnapshotDAO creates a prune snapshot DAO.+func NewSyncPruneSnapshotDAO(db *gorm.DB) *SyncPruneSnapshotDAO {+ return &SyncPruneSnapshotDAO{db: db}+}++// InsertBatch writes one prune snapshot batch.+func (d *SyncPruneSnapshotDAO) InsertBatch(ctx context.Context, taskID string, documents []syncerconnector.SlimDocument) error {+ if len(documents) == 0 {+ return nil+ }+ rows := make([]SyncPruneSnapshot, 0, len(documents))+ for _, doc := range documents {+ rows = append(rows, SyncPruneSnapshot{TaskID: taskID, SourceDocID: doc.SourceID})+ }+ return d.db.WithContext(ctx).Create(&rows).Error+}++// ListSourceIDs returns all source IDs for a task snapshot.+func (d *SyncPruneSnapshotDAO) ListSourceIDs(ctx context.Context, taskID string) ([]string, error) {+ var ids []string+ err := d.db.WithContext(ctx).Model(&SyncPruneSnapshot{}).Where("task_id = ?", taskID).Pluck("source_doc_id", &ids).Error+ return ids, err+}++// Clear removes all snapshot rows for a task.+func (d *SyncPruneSnapshotDAO) Clear(ctx context.Context, taskID string) error {+ return d.db.WithContext(ctx).Where("task_id = ?", taskID).Delete(&SyncPruneSnapshot{}).Error+}diff --git a/internal/dao/sync_task.go b/internal/dao/sync_task.gonew file mode 100644index 0000000000..abb0961a57--- /dev/null+++ b/internal/dao/sync_task.go@@ -0,0 +1,376 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "errors"+ "ragflow/internal/entity"+ "ragflow/internal/utility"+ "time"++ "gorm.io/gorm"+ "gorm.io/gorm/clause"+)++const (+ // TaskTypeSync is the Python-compatible SYNC task type.+ TaskTypeSync = "sync"+ // TaskTypePrune is the Python-compatible PRUNE task type.+ TaskTypePrune = "prune"+ // SyncStatusSchedule is the Python TaskStatus.SCHEDULE value for sync_logs.+ SyncStatusSchedule = "5"+ // SyncStatusRunning is the Python TaskStatus.RUNNING value for sync_logs.+ SyncStatusRunning = "1"+ // SyncStatusCancel is the Python TaskStatus.CANCEL value for sync_logs.+ SyncStatusCancel = "2"+ // SyncStatusDone is the Python TaskStatus.DONE value for sync_logs.+ SyncStatusDone = "3"+ // SyncStatusFail is the Python TaskStatus.FAIL value for sync_logs.+ SyncStatusFail = "4"+)++// SyncTask is the Python-compatible sync task row.+type SyncTask struct {+ entity.SyncLogs `gorm:"embedded"`+}++// TableName returns the Python-compatible sync task table.+func (SyncTask) TableName() string {+ return "sync_logs"+}++// SyncTaskContext contains every database row required to execute one task.+type SyncTaskContext struct {+ Task entity.SyncLogs+ Connector entity.Connector+ Connector2Kb entity.Connector2Kb+ Knowledgebase entity.Knowledgebase+}++// ConnectorSource returns the connector source name.+func (c SyncTaskContext) ConnectorSource() string {+ return c.Connector.Source+}++// SyncTaskDAO reads and updates sync_logs tasks.+type SyncTaskDAO struct {+ db *gorm.DB+}++// NewSyncTaskDAO creates a syncer task DAO.+func NewSyncTaskDAO(db *gorm.DB) *SyncTaskDAO {+ if db == nil {+ db = GetDB()+ }+ return &SyncTaskDAO{db: db}+}++// DB returns the DAO database handle.+func (d *SyncTaskDAO) DB() *gorm.DB {+ return d.db+}++type dueSyncTaskRow struct {+ entity.SyncLogs+ ConnectorRefreshFreq int64 `gorm:"column:connector_refresh_freq"`+ ConnectorPruneFreq int64 `gorm:"column:connector_prune_freq"`+ ConnectorConfig entity.JSONMap `gorm:"column:connector_config"`+}++// ListDueTasks returns due schedule tasks across SYNC and PRUNE.+func (d *SyncTaskDAO) ListDueTasks(ctx context.Context, now time.Time, limit int) ([]entity.SyncLogs, error) {+ var rows []dueSyncTaskRow+ if err := d.db.WithContext(ctx).+ Model(&entity.SyncLogs{}).+ Select("sync_logs.*, connector.refresh_freq AS connector_refresh_freq, connector.prune_freq AS connector_prune_freq, connector.config AS connector_config").+ Joins("JOIN connector ON sync_logs.connector_id = connector.id").+ Joins("JOIN connector2kb ON sync_logs.connector_id = connector2kb.connector_id AND sync_logs.kb_id = connector2kb.kb_id").+ Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").+ Where("sync_logs.status = ? AND connector.status = ? AND sync_logs.task_type IN ?", SyncStatusSchedule, SyncStatusSchedule, []string{TaskTypeSync, TaskTypePrune}).+ Order("sync_logs.update_time DESC").+ Limit(limit * 4).+ Scan(&rows).Error; err != nil {+ return nil, err+ }++ tasks := make([]entity.SyncLogs, 0, limit)+ for _, row := range rows {+ if !isDue(row.SyncLogs, row.ConnectorRefreshFreq, row.ConnectorPruneFreq, row.ConnectorConfig, now) {+ continue+ }+ tasks = append(tasks, row.SyncLogs)+ if len(tasks) >= limit {+ break+ }+ }+ return tasks, nil+}++// ClaimTask conditionally marks a scheduled task as running.+func (d *SyncTaskDAO) ClaimTask(ctx context.Context, taskID string, now time.Time) (bool, error) {+ result := d.db.WithContext(ctx).Model(&entity.SyncLogs{}).+ Where("id = ? AND status = ?", taskID, SyncStatusSchedule).+ Updates(map[string]any{"status": SyncStatusRunning, "time_started": now})+ if result.Error != nil {+ return false, result.Error+ }+ return result.RowsAffected == 1, nil+}++// GetTaskContext loads a task with connector, mapping, and knowledgebase rows.+func (d *SyncTaskDAO) GetTaskContext(ctx context.Context, taskID string) (SyncTaskContext, error) {+ var task entity.SyncLogs+ if err := d.db.WithContext(ctx).Where("id = ?", taskID).First(&task).Error; err != nil {+ return SyncTaskContext{}, err+ }+ // get connector+ var connector entity.Connector+ if err := d.db.WithContext(ctx).Where("id = ?", task.ConnectorID).First(&connector).Error; err != nil {+ return SyncTaskContext{}, err+ }+ // get relation+ var connector2Kb entity.Connector2Kb+ if err := d.db.WithContext(ctx).Where("connector_id = ? AND kb_id = ?", task.ConnectorID, task.KbID).First(&connector2Kb).Error; err != nil {+ return SyncTaskContext{}, err+ }+ // get KB+ var kb entity.Knowledgebase+ if err := d.db.WithContext(ctx).Where("id = ?", task.KbID).First(&kb).Error; err != nil {+ return SyncTaskContext{}, err+ }++ return SyncTaskContext{Task: task, Connector: connector, Connector2Kb: connector2Kb, Knowledgebase: kb}, nil+}++// MarkConnectorRunning marks a connector running.+func (d *SyncTaskDAO) MarkConnectorRunning(ctx context.Context, connectorID string) error {+ return d.db.WithContext(ctx).Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusRunning).Error+}++// RescheduleClaimed puts a claimed task back into schedule state.+func (d *SyncTaskDAO) RescheduleClaimed(ctx context.Context, taskID string) error {+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var task entity.SyncLogs+ if err := tx.WithContext(ctx).Where("id = ? AND status = ?", taskID, SyncStatusRunning).First(&task).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil+ }+ return err+ }+ if err := tx.Model(&entity.SyncLogs{}).+ Where("id = ? AND status = ?", taskID, SyncStatusRunning).+ Update("status", SyncStatusSchedule).Error; err != nil {+ return err+ }+ return tx.Model(&entity.Connector{}).Where("id = ? AND status = ?", task.ConnectorID, SyncStatusRunning).Update("status", SyncStatusSchedule).Error+ })+}++// FailTask marks a task failed without advancing its poll waterline.+func (d *SyncTaskDAO) FailTask(ctx context.Context, taskID, connectorID, message string, errorCount int64) error {+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.Model(&entity.SyncLogs{}).Where("id = ?", taskID).Updates(map[string]any{+ "status": SyncStatusFail,+ "error_msg": message,+ "error_count": errorCount,+ }).Error; err != nil {+ return err+ }+ if connectorID == "" {+ return nil+ }+ return tx.Model(&entity.Connector{}).Where("id = ?", connectorID).Update("status", SyncStatusFail).Error+ })+}++// CompleteSyncTask marks SYNC done and creates the next schedule task.+func (d *SyncTaskDAO) CompleteSyncTask(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, newDocs, totalDocs, errorCount int64, errorMsg string) error {+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.Model(&entity.SyncLogs{}).Where("id = ?", taskContext.Task.ID).Updates(map[string]any{+ "status": SyncStatusDone,+ "poll_range_end": pollRangeEnd,+ "new_docs_indexed": newDocs,+ "total_docs_indexed": gorm.Expr("total_docs_indexed + ?", totalDocs),+ "error_msg": errorMsg,+ "error_count": errorCount,+ }).Error; err != nil {+ return err+ }+ if err := tx.Model(&entity.Connector{}).Where("id = ?", taskContext.Connector.ID).Update("status", SyncStatusDone).Error; err != nil {+ return err+ }++ return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypeSync, false, &pollRangeEnd, taskContext.Task.TotalDocsIndexed+totalDocs)+ })+}++// CompletePruneTask marks PRUNE done and creates the next schedule task.+func (d *SyncTaskDAO) CompletePruneTask(ctx context.Context, taskContext SyncTaskContext, removed int64) error {+ return d.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.Model(&entity.SyncLogs{}).Where("id = ?", taskContext.Task.ID).Updates(map[string]any{+ "status": SyncStatusDone,+ "docs_removed_from_index": gorm.Expr("docs_removed_from_index + ?", removed),+ }).Error; err != nil {+ return err+ }+ if err := tx.Model(&entity.Connector{}).Where("id = ?", taskContext.Connector.ID).Update("status", SyncStatusDone).Error; err != nil {+ return err+ }+ if !syncConnectorConfigBool(taskContext.Connector.Config, "sync_deleted_files") {+ return nil+ }+ return createScheduledTask(ctx, tx, taskContext.Connector.ID, taskContext.Knowledgebase.ID, TaskTypePrune, false, nil, taskContext.Task.TotalDocsIndexed)+ })+}++// RecoverStaleRunning restores timed-out running tasks to schedule.+func (d *SyncTaskDAO) RecoverStaleRunning(ctx context.Context, now time.Time) (int64, error) {+ type staleRunningTaskRow struct {+ ID string `gorm:"column:id"`+ ConnectorID string `gorm:"column:connector_id"`+ TimeStarted *time.Time `gorm:"column:time_started"`+ ConnectorTimeoutSecs int64 `gorm:"column:connector_timeout_secs"`+ }+ var rows []staleRunningTaskRow+ if err := d.db.WithContext(ctx).+ Model(&entity.SyncLogs{}).+ Select("sync_logs.id, sync_logs.connector_id, sync_logs.time_started, connector.timeout_secs AS connector_timeout_secs").+ Joins("JOIN connector ON sync_logs.connector_id = connector.id").+ Where("sync_logs.status = ?", SyncStatusRunning).+ Scan(&rows).Error; err != nil {+ return 0, err+ }+ var recovered int64+ connectorIDs := map[string]struct{}{}++ for _, row := range rows {+ if row.TimeStarted == nil {+ continue+ }+ timeout := time.Duration(row.ConnectorTimeoutSecs) * time.Second+ if timeout <= 0 {+ timeout = time.Hour+ }+ if row.TimeStarted.Add(timeout).After(now) {+ continue+ }+ if err := d.RescheduleClaimed(ctx, row.ID); err != nil {+ return recovered, err+ }+ recovered+++ connectorIDs[row.ConnectorID] = struct{}{}+ }++ if recovered == 0 {+ return 0, nil+ }+ ids := make([]string, 0, len(connectorIDs))++ for connectorID := range connectorIDs {+ ids = append(ids, connectorID)+ }++ return recovered, d.db.WithContext(ctx).Model(&entity.Connector{}).Where("id IN ? AND status = ?", ids, SyncStatusRunning).Update("status", SyncStatusSchedule).Error+}++// createScheduledTask creates the next Python-compatible scheduled task.+func createScheduledTask(ctx context.Context, tx *gorm.DB, connectorID, kbID, taskType string, fromBeginning bool, pollRangeStart *time.Time, totalDocsIndexed int64) error {+ var lockRow entity.Connector2Kb+ query := tx.WithContext(ctx)+ if tx.Dialector.Name() != "sqlite" {+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})+ }+ if err := query.+ Where("connector_id = ? AND kb_id = ?", connectorID, kbID).+ First(&lockRow).Error; err != nil {+ return err+ }++ var existing int64+ if err := tx.WithContext(ctx).Model(&entity.SyncLogs{}).+ Where("connector_id = ? AND kb_id = ? AND task_type = ? AND status = ?", connectorID, kbID, taskType, SyncStatusSchedule).+ Count(&existing).Error; err != nil {+ return err+ }+ if existing > 0 {+ return nil+ }++ reindex := "0"+ if fromBeginning {+ reindex = "1"+ }++ now := time.Now().Local()+ return tx.WithContext(ctx).Create(&entity.SyncLogs{+ ID: utility.GenerateToken(),+ ConnectorID: connectorID,+ KbID: kbID,+ TaskType: taskType,+ Status: SyncStatusSchedule,+ FromBeginning: &reindex,+ PollRangeStart: pollRangeStart,+ TimeStarted: &now,+ ErrorMsg: "",+ TotalDocsIndexed: totalDocsIndexed,+ }).Error+}++// isDue applies refresh_freq and prune_freq scheduling semantics.+func isDue(task entity.SyncLogs, refreshFreq, pruneFreq int64, config map[string]any, now time.Time) bool {+ if task.UpdateDate == nil {+ return true+ }+ var freqMinutes int64+ switch task.TaskType {+ case TaskTypeSync:+ freqMinutes = refreshFreq+ case TaskTypePrune:+ if !syncConnectorConfigBool(config, "sync_deleted_files") {+ return false+ }+ freqMinutes = pruneFreq+ default:+ return false+ }+ if freqMinutes <= 0 {+ return true+ }+ return task.UpdateDate.Before(now.Add(-time.Duration(freqMinutes) * time.Minute))+}++// syncConnectorConfigBool reads a Python JSON bool/string flag.+func syncConnectorConfigBool(config map[string]any, key string) bool {+ value, ok := config[key]+ if !ok {+ return false+ }+ switch typed := value.(type) {+ case bool:+ return typed+ case string:+ return typed == "1" || typed == "true" || typed == "TRUE"+ default:+ return false+ }+}++// IsNotFound reports whether an error is a gorm not found error.+func IsNotFound(err error) bool {+ return errors.Is(err, gorm.ErrRecordNotFound)+}diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex 2a69b61b3d..8df895966b 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -40,6 +40,8 @@ import ("ragflow/internal/service""ragflow/internal/service/file""ragflow/internal/storage"+ syncerconnector "ragflow/internal/syncer/connector"+ "ragflow/internal/utility")// recordingTaskPublisher implements service.TaskPublisher and records published messages.@@ -55,7 +57,8 @@ func (r *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.T}type fakeUploadStorage struct {- objects map[string][]byte+ objects map[string][]byte+ afterPut func()}func newFakeUploadStorage() *fakeUploadStorage {@@ -67,6 +70,9 @@ func (f *fakeUploadStorage) Health(_ context.Context) bool { return true }func (f *fakeUploadStorage) key(bucket, fnm string) string { return bucket + "/" + fnm }func (f *fakeUploadStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {f.objects[f.key(bucket, fnm)] = append([]byte(nil), binary...)+ if f.afterPut != nil {+ f.afterPut()+ }return nil}func (f *fakeUploadStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {@@ -730,6 +736,121 @@ func TestContentHashHex_MatchesPythonXXH128(t *testing.T) {}}+func TestSyncDocumentUpsertRemovesStagedBlobWhenInsertFails(t *testing.T) {+ ctx := context.Background()+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ mockStorage := newFakeUploadStorage()+ factory := storage.GetStorageFactory()+ origStorage := factory.GetStorage()+ factory.SetStorage(mockStorage)+ t.Cleanup(func() { factory.SetStorage(origStorage) })++ svc := testDocumentService(t)+ _, err := svc.Upsert(ctx, service.DocumentUpsertInput{+ TaskContext: service.SyncTaskContext{+ Connector: entity.Connector{TenantID: "tenant-1"},+ Knowledgebase: entity.Knowledgebase{+ ID: "missing-kb",+ TenantID: "tenant-1",+ Name: "Missing KB",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ },+ },+ SourceType: "github",+ DocumentID: "doc-sync-insert",+ SourceDocument: syncerconnector.SourceDocument{+ SourceID: "source-1",+ SemanticIdentifier: "source",+ Extension: ".txt",+ Blob: []byte("new content"),+ },+ })+ if err == nil {+ t.Fatalf("expected insert failure")+ }+ for key := range mockStorage.objects {+ if strings.Contains(key, "/.staged/") {+ t.Fatalf("expected staged object cleanup, found %s", key)+ }+ }+}++func TestSyncDocumentUpsertRemovesStagedBlobWhenUpdateFails(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ mockStorage := newFakeUploadStorage()+ factory := storage.GetStorageFactory()+ origStorage := factory.GetStorage()+ factory.SetStorage(mockStorage)+ t.Cleanup(func() { factory.SetStorage(origStorage) })++ activeLocation := "sync/github/doc-sync-update.txt"+ if err := mockStorage.Put(context.Background(), "kb-sync", activeLocation, []byte("old content")); err != nil {+ t.Fatalf("seed active object: %v", err)+ }+ oldName := "old.txt"+ oldHash := "old-hash"+ if err := db.Create(&entity.Document{+ ID: "doc-sync-update",+ KbID: "kb-sync",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ SourceType: "github",+ Type: string(utility.FileTypeTXT),+ CreatedBy: "tenant-1",+ Name: &oldName,+ Location: &activeLocation,+ Size: int64(len("old content")),+ Suffix: "txt",+ ContentHash: &oldHash,+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ svc := testDocumentService(t)+ mockStorage.afterPut = cancel+ _, err := svc.Upsert(ctx, service.DocumentUpsertInput{+ TaskContext: service.SyncTaskContext{+ Connector: entity.Connector{TenantID: "tenant-1"},+ Knowledgebase: entity.Knowledgebase{+ ID: "kb-sync",+ TenantID: "tenant-1",+ Name: "Sync KB",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ },+ },+ SourceType: "github",+ DocumentID: "doc-sync-update",+ SourceDocument: syncerconnector.SourceDocument{+ SourceID: "source-1",+ SemanticIdentifier: "source",+ Extension: ".txt",+ Blob: []byte("new content"),+ },+ })+ if err == nil {+ t.Fatalf("expected update failure")+ }+ got, err := mockStorage.Get(context.Background(), "kb-sync", activeLocation)+ if err != nil {+ t.Fatalf("active object was removed: %v", err)+ }+ if string(got) != "old content" {+ t.Fatalf("active object overwritten: %q", got)+ }+ for key := range mockStorage.objects {+ if strings.Contains(key, "/.staged/") {+ t.Fatalf("expected staged object cleanup, found %s", key)+ }+ }+}+func TestUploadLocalDocuments_MirrorsPythonCoreFields(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/document/sync_document.go b/internal/service/document/sync_document.gonew file mode 100644index 0000000000..56ab4a5963--- /dev/null+++ b/internal/service/document/sync_document.go@@ -0,0 +1,315 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package document++import (+ "context"+ "errors"+ "fmt"+ "path/filepath"+ "regexp"+ "strings"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/service"+ "ragflow/internal/storage"+ "ragflow/internal/utility"++ "gorm.io/gorm"+)++var syncFilenameUnsafeRE = regexp.MustCompile(`[\\/:*?"<>|]+`)++// Upsert stores one normalized source document.+func (s *DocumentService) Upsert(ctx context.Context, input service.DocumentUpsertInput) (service.DocumentUpsertResult, error) {+ if input.DocumentID == "" {+ return service.DocumentUpsertResult{}, fmt.Errorf("document id is empty")+ }++ // get base info+ kb := &input.TaskContext.Knowledgebase+ tenantID := input.TaskContext.Connector.TenantID+ // return normalized filename+ filename := syncDocumentFilename(input.SourceDocument.SemanticIdentifier, input.SourceDocument.Extension, input.SourceDocument.SourceID)+ filetype := utility.FilenameType(filename)+ if filetype == utility.FileTypeOTHER {+ return service.DocumentUpsertResult{}, fmt.Errorf("document type %q is not supported", filename)+ }++ // get storage+ storageImpl := storage.GetStorageFactory().GetStorage()+ if storageImpl == nil {+ return service.DocumentUpsertResult{}, fmt.Errorf("storage not initialized")+ }++ // get content hash+ contentHash := input.SourceDocument.Fingerprint+ if contentHash == "" {+ contentHash = contentHashHex(input.SourceDocument.Blob)+ }++ // if the 'file' is existing+ existing, err := s.documentDAO.GetByID(ctx, dao.DB, input.DocumentID)+ if err != nil && !errorsIsRecordNotFound(err) {+ return service.DocumentUpsertResult{}, err+ }+ if existing != nil && existing.ContentHash != nil && *existing.ContentHash == contentHash {+ if err := s.ensureSyncDocumentPostWrite(ctx, input, existing); err != nil {+ return service.DocumentUpsertResult{}, err+ }+ return service.DocumentUpsertResult{DocID: input.DocumentID, Action: service.DocumentActionSkipped}, nil+ }++ // store location+ location := syncDocumentStagedLocation(input.SourceType, input.DocumentID, filename)+ cleanupLocation := true+ defer func() {+ if cleanupLocation {+ _ = storageImpl.Remove(context.WithoutCancel(ctx), kb.ID, location)+ }+ }()+ if err = storageImpl.Put(ctx, kb.ID, location, input.SourceDocument.Blob); err != nil {+ cleanupLocation = false+ return service.DocumentUpsertResult{}, err+ }++ // not existing, insert a new 'file'+ if existing == nil {+ result, insertErr := s.insertSyncDocument(ctx, input, filename, location, string(filetype), contentHash, tenantID)+ if insertErr == nil {+ cleanupLocation = false+ }+ return result, insertErr+ }+ // just update the file+ result, updateErr := s.updateSyncDocument(ctx, input, existing, filename, location, string(filetype), contentHash)+ if updateErr == nil {+ cleanupLocation = false+ }+ return result, updateErr+}++// insertSyncDocument inserts a new synced document and file-manager link.+func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.DocumentUpsertInput, filename, location, filetype, contentHash, tenantID string) (service.DocumentUpsertResult, error) {+ // base info+ kb := &input.TaskContext.Knowledgebase+ kbFolder, err := s.ensureKBFolder(ctx, kb, tenantID)+ if err != nil {+ return service.DocumentUpsertResult{}, err+ }++ // create the 'doc'+ doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, copyJSONMap(kb.ParserConfig), input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)+ doc.ID = input.DocumentID++ // put 'file' in mysql `document`+ if err = s.InsertDocument(doc); err != nil {+ return service.DocumentUpsertResult{}, err+ }+ if err = s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {+ return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)+ }+ // link 'file' 2 'file_system'+ if err = s.addFileFromKB(ctx, doc, kbFolder.ID, kb.TenantID); err != nil {+ return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)+ }+ // write metadata and do `auto_parse` (or not)+ if err = s.afterSyncDocumentUpsert(ctx, input, doc, false); err != nil {+ return service.DocumentUpsertResult{}, err+ }++ return service.DocumentUpsertResult{DocID: doc.ID, Action: service.DocumentActionAdded}, nil+}++// updateSyncDocument updates an existing synced document and file-manager row.+func (s *DocumentService) updateSyncDocument(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, filename, location, filetype, contentHash string) (service.DocumentUpsertResult, error) {+ suffix := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".")+ updates := map[string]any{+ "name": filename,+ "size": int64(len(input.SourceDocument.Blob)),+ "type": filetype,+ "suffix": suffix,+ "source_type": input.SourceType,+ }+ if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, updates); err != nil {+ return service.DocumentUpsertResult{}, err+ }++ doc.Name = &filename+ doc.Size = int64(len(input.SourceDocument.Blob))+ doc.Type = filetype+ doc.Suffix = suffix+ doc.SourceType = input.SourceType++ if err := s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {+ return service.DocumentUpsertResult{}, err+ }+ // update new updated file+ if err := s.updateSyncDocumentFile(ctx, input, doc); err != nil {+ return service.DocumentUpsertResult{}, err+ }+ // write metadata and do `auto_parse`+ if err := s.afterSyncDocumentUpsert(ctx, input, doc, true); err != nil {+ return service.DocumentUpsertResult{}, err+ }+ return service.DocumentUpsertResult{DocID: doc.ID, Action: service.DocumentActionUpdated}, nil+}++// publishSyncDocument publishes the staged object key after the document row exists.+func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.Document, location, contentHash string) error {+ if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{+ "location": location,+ "content_hash": contentHash,+ }); err != nil {+ return err+ }+ doc.Location = &location+ doc.ContentHash = &contentHash+ return nil+}++// afterSyncDocumentUpsert writes metadata and optionally enqueues parsing.+func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, rerun bool) error {+ // write metadata+ if len(input.SourceDocument.Metadata) > 0 && s.docEngine != nil {+ if err := s.SetDocumentMetadata(ctx, doc.ID, input.SourceDocument.Metadata); err != nil {+ return err+ }+ }+ // do auto_parse if enabled+ if !input.AutoParse {+ return nil+ }+ return s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{RerunWithDelete: rerun})+}++// ensureSyncDocumentPostWrite retries dependent work before an unchanged document is skipped.+func (s *DocumentService) ensureSyncDocumentPostWrite(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document) error {+ if err := s.updateSyncDocumentFile(ctx, input, doc); err != nil {+ return err+ }+ if len(input.SourceDocument.Metadata) > 0 && s.docEngine != nil {+ if err := s.SetDocumentMetadata(ctx, doc.ID, input.SourceDocument.Metadata); err != nil {+ return err+ }+ }+ if !input.AutoParse {+ return nil+ }+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return err+ }+ if task != nil {+ return nil+ }+ return s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{})+}++// updateSyncDocumentFile updates the file-manager row linked to a synced document.+func (s *DocumentService) updateSyncDocumentFile(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document) error {+ mappings, err := s.file2DocumentDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return err+ }++ // no updates+ if len(mappings) == 0 || mappings[0].FileID == nil {+ kbFolder, folderErr := s.ensureKBFolder(ctx, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID)+ if folderErr != nil {+ return folderErr+ }+ return s.addFileFromKB(ctx, doc, kbFolder.ID, input.TaskContext.Knowledgebase.TenantID)+ }++ name := ""+ if doc.Name != nil {+ name = *doc.Name+ }++ location := ""+ if doc.Location != nil {+ location = *doc.Location+ }++ return s.fileDAO.UpdateByID(ctx, dao.DB, *mappings[0].FileID, map[string]interface{}{+ "name": name,+ "location": location,+ "size": doc.Size,+ "type": doc.Type,+ })+}++// syncDocumentFilename returns a bounded filename for a synced source document.+func syncDocumentFilename(name, extension, fallback string) string {+ name = strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(name, "_"))+ if name == "" {+ name = strings.TrimSpace(syncFilenameUnsafeRE.ReplaceAllString(fallback, "_"))+ }+ if name == "" {+ name = "document"+ }++ extension = strings.TrimSpace(extension)+ if extension == "" {+ extension = ".txt"+ }++ if !strings.HasPrefix(extension, ".") {+ extension = "." + extension+ }+ if !strings.HasSuffix(strings.ToLower(name), strings.ToLower(extension)) {+ name += extension+ }++ if len(name) <= 255 {+ return name+ }++ ext := filepath.Ext(name)+ baseLimit := 255 - len(ext)+ if baseLimit < 1 {+ return name[:255]+ }++ return name[:baseLimit] + ext+}++// syncDocumentStagedLocation returns a unique object-storage key for unpublished synced content.+func syncDocumentStagedLocation(sourceType, docID, filename string) string {+ ext := filepath.Ext(filename)+ sourceType = strings.Trim(strings.ReplaceAll(sourceType, "/", "_"), "_")+ if sourceType == "" {+ sourceType = "sync"+ }+ return fmt.Sprintf("sync/%s/.staged/%s/%s%s", sourceType, utility.GenerateToken(), docID, ext)+}++// copyJSONMap returns a shallow copy of a JSON map.+func copyJSONMap(value entity.JSONMap) entity.JSONMap {+ out := entity.JSONMap{}+ for k, v := range value {+ out[k] = v+ }+ return out+}++// errorsIsRecordNotFound reports whether an error is GORM's not-found error.+func errorsIsRecordNotFound(err error) bool {+ return errors.Is(err, gorm.ErrRecordNotFound)+}diff --git a/internal/service/sync_document_id_resolver.go b/internal/service/sync_document_id_resolver.gonew file mode 100644index 0000000000..b9605badc7--- /dev/null+++ b/internal/service/sync_document_id_resolver.go@@ -0,0 +1,121 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "encoding/hex"+ "ragflow/internal/dao"+ "ragflow/internal/entity"++ "github.com/zeebo/xxh3"+)++// Hash128 api.utils.common.hash128.+func Hash128(data string) string {+ sum := xxh3.Hash128([]byte(data)).Bytes()+ return hex.EncodeToString(sum[:])+}++// ResolvedDocumentID contains legacy and new connector document IDs.+type ResolvedDocumentID struct {+ DocID string+ LegacyID string+ NewID string+ StoredFingerprint string+}++// DocumentStore reads existing connector documents.+type DocumentStore interface {+ ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error)+ GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error)+}++// DocumentIDResolver applies Python-compatible legacy/new ID rules.+type DocumentIDResolver struct {+ store DocumentStore+}++// NewDocumentIDResolver creates a document ID resolver.+func NewDocumentIDResolver(store DocumentStore) *DocumentIDResolver {+ return &DocumentIDResolver{store: store}+}++// Resolve chooses the legacy ID when already present in the KB.+func (r *DocumentIDResolver) Resolve(ctx context.Context, kbID, connectorID, sourceType, sourceID string) (ResolvedDocumentID, error) {+ legacyID := Hash128(connectorID + ":" + sourceID)+ newID := Hash128(kbID + ":" + connectorID + ":" + sourceID)++ fingerprints, err := r.store.GetFingerprintsByIDs(ctx, kbID, sourceType, []string{legacyID, newID})+ if err != nil {+ return ResolvedDocumentID{}, err+ }++ docID := newID+ if _, ok := fingerprints[legacyID]; ok {+ docID = legacyID+ }+ stored := fingerprints[legacyID]+ if stored == "" {+ stored = fingerprints[newID]+ }++ return ResolvedDocumentID{DocID: docID, LegacyID: legacyID, NewID: newID, StoredFingerprint: stored}, nil+}++// GormDocumentStore reads documents from the current GORM database.+type GormDocumentStore struct{}++// NewGormDocumentStore creates a GORM-backed document store.+func NewGormDocumentStore() *GormDocumentStore {+ return &GormDocumentStore{}+}++// ListIDs returns existing connector document IDs in a KB.+func (s *GormDocumentStore) ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error) {+ var docs []entity.Document+ if err := dao.GetDB().WithContext(ctx).Select("id").Where("kb_id = ? AND source_type = ?", kbID, sourceType).Find(&docs).Error; err != nil {+ return nil, err+ }++ result := make(map[string]struct{}, len(docs))+ for _, doc := range docs {+ result[doc.ID] = struct{}{}+ }++ return result, nil+}++// GetFingerprintsByIDs returns fingerprints for the requested candidate IDs.+func (s *GormDocumentStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {+ var docs []entity.Document+ if err := dao.GetDB().WithContext(ctx).+ Select("id", "content_hash").+ Where("kb_id = ? AND source_type = ? AND id IN ?", kbID, sourceType, ids).+ Find(&docs).Error; err != nil {+ return nil, err+ }+ result := make(map[string]string, len(docs))+ for _, doc := range docs {+ if doc.ContentHash == nil {+ result[doc.ID] = ""+ continue+ }+ result[doc.ID] = *doc.ContentHash+ }+ return result, nil+}diff --git a/internal/service/sync_document_sink.go b/internal/service/sync_document_sink.gonew file mode 100644index 0000000000..d40bff72ed--- /dev/null+++ b/internal/service/sync_document_sink.go@@ -0,0 +1,88 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "errors"+ syncerconnector "ragflow/internal/syncer/connector"+ "time"+)++const (+ // DocumentActionAdded means a document was newly inserted.+ DocumentActionAdded = "added"+ // DocumentActionUpdated means a document was changed.+ DocumentActionUpdated = "updated"+ // DocumentActionSkipped means a document was unchanged.+ DocumentActionSkipped = "skipped"+)++// DocumentUpsertInput describes one normalized source document write.+type DocumentUpsertInput struct {+ TaskContext SyncTaskContext+ SourceType string+ DocumentID string+ LegacyID string+ NewID string+ SourceDocument syncerconnector.SourceDocument+ AutoParse bool+}++// DocumentUpsertResult describes one sink write result.+type DocumentUpsertResult struct {+ DocID string+ Action string+}++// DocumentSink stores one normalized source document.+type DocumentSink interface {+ // Upsert stores one normalized source document.+ Upsert(ctx context.Context, input DocumentUpsertInput) (DocumentUpsertResult, error)+}++// RetryableError marks a failure as safe to retry.+type RetryableError struct {+ Err error+ After time.Duration+ Temporary bool+ StatusCode int+}++// Error returns the wrapped error message.+func (e *RetryableError) Error() string {+ if e == nil || e.Err == nil {+ return "retryable sync error"+ }+ return e.Err.Error()+}++// Unwrap returns the wrapped error.+func (e *RetryableError) Unwrap() error {+ if e == nil {+ return nil+ }+ return e.Err+}++// IsRetryable reports whether an error should be retried.+func IsRetryable(err error) bool {+ if _, ok := errors.AsType[*RetryableError](err); ok {+ return true+ }+ return false+}diff --git a/internal/service/sync_prune_service.go b/internal/service/sync_prune_service.gonew file mode 100644index 0000000000..107464ee28--- /dev/null+++ b/internal/service/sync_prune_service.go@@ -0,0 +1,99 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "errors"+ "ragflow/internal/dao"+ syncerconnector "ragflow/internal/syncer/connector"+)++// ErrSyncDocumentDeleterNotConfigured reports a missing production delete service.+var ErrSyncDocumentDeleterNotConfigured = errors.New("sync document deleter is not configured")++// SyncDocumentDeleter removes documents through the owning business service.+type SyncDocumentDeleter interface {+ // DeleteDocument removes a document and dependent business state.+ DeleteDocument(ctx context.Context, docID string) error+}++// SyncPruneService owns prune snapshots and stale document deletion.+type SyncPruneService struct {+ snapshotDAO *dao.SyncPruneSnapshotDAO+ deleter SyncDocumentDeleter+ store DocumentStore+}++// NewSyncPruneService creates a prune service.+func NewSyncPruneService(snapshotDAO *dao.SyncPruneSnapshotDAO, deleter SyncDocumentDeleter, store DocumentStore) *SyncPruneService {+ if store == nil {+ store = NewGormDocumentStore()+ }+ return &SyncPruneService{snapshotDAO: snapshotDAO, deleter: deleter, store: store}+}++// ClearSnapshot removes temporary snapshot rows.+func (s *SyncPruneService) ClearSnapshot(ctx context.Context, taskID string) error {+ return s.snapshotDAO.Clear(ctx, taskID)+}++// AddSnapshotBatch writes one slim snapshot batch.+func (s *SyncPruneService) AddSnapshotBatch(ctx context.Context, taskID string, documents []syncerconnector.SlimDocument) error {+ return s.snapshotDAO.InsertBatch(ctx, taskID, documents)+}++// DeleteStale removes documents missing from the complete source snapshot.+func (s *SyncPruneService) DeleteStale(ctx context.Context, taskContext SyncTaskContext) (int64, error) {+ sourceIDs, err := s.snapshotDAO.ListSourceIDs(ctx, taskContext.Task.ID)+ if err != nil {+ return 0, err+ }+ sourceType := SourceType(taskContext.Connector.Source, taskContext.Connector.ID)+ retain := RetainDocumentIDs(taskContext.Knowledgebase.ID, taskContext.Connector.ID, sourceIDs)+ existing, err := s.store.ListIDs(ctx, taskContext.Knowledgebase.ID, sourceType)+ if err != nil {+ return 0, err+ }+ var removed int64+ for docID := range existing {+ if _, ok := retain[docID]; ok {+ continue+ }+ if s.deleter == nil {+ return removed, ErrSyncDocumentDeleterNotConfigured+ }+ if err = s.deleter.DeleteDocument(ctx, docID); err != nil {+ return removed, err+ }+ removed+++ }+ if err = s.snapshotDAO.Clear(ctx, taskContext.Task.ID); err != nil {+ return removed, err+ }+ return removed, nil+}++// RetainDocumentIDs calculates PRUNE retain IDs using both legacy and new schemes.+func RetainDocumentIDs(kbID, connectorID string, sourceIDs []string) map[string]struct{} {+ retain := make(map[string]struct{}, len(sourceIDs)*2)+ for _, sourceID := range sourceIDs {+ retain[Hash128(connectorID+":"+sourceID)] = struct{}{}+ retain[Hash128(kbID+":"+connectorID+":"+sourceID)] = struct{}{}+ }+ return retain+}diff --git a/internal/service/sync_task_service.go b/internal/service/sync_task_service.gonew file mode 100644index 0000000000..5d383ec1d0--- /dev/null+++ b/internal/service/sync_task_service.go@@ -0,0 +1,157 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "fmt"+ "ragflow/internal/dao"+ "strings"+ "time"+)++const (+ // TaskTypeSync is the Python-compatible SYNC task type.+ TaskTypeSync = dao.TaskTypeSync+ // TaskTypePrune is the Python-compatible PRUNE task type.+ TaskTypePrune = dao.TaskTypePrune+)++// SyncTaskContext contains the database rows needed to execute a task.+type SyncTaskContext = dao.SyncTaskContext++// SyncStats accumulates task-level document results.+type SyncStats struct {+ Added int64+ Updated int64+ Skipped int64+ ErrorCount int64+ ErrorMsg string+}++// Add merges another stats value.+func (s *SyncStats) Add(other SyncStats) {+ s.Added += other.Added+ s.Updated += other.Updated+ s.Skipped += other.Skipped+ s.ErrorCount += other.ErrorCount+ if other.ErrorMsg != "" {+ if s.ErrorMsg != "" {+ s.ErrorMsg += "\n"+ }+ s.ErrorMsg += other.ErrorMsg+ }+}++// AddResult merges one document result.+func (s *SyncStats) AddResult(result DocumentUpsertResult) {+ switch result.Action {+ case DocumentActionAdded:+ s.Added+++ case DocumentActionUpdated:+ s.Updated+++ case DocumentActionSkipped:+ s.Skipped+++ }+}++// SyncTaskService contains sync task business updates.+type SyncTaskService struct {+ taskDAO *dao.SyncTaskDAO+}++// NewSyncTaskService creates a task service.+func NewSyncTaskService(taskDAO *dao.SyncTaskDAO) *SyncTaskService {+ return &SyncTaskService{taskDAO: taskDAO}+}++// ListDueTasks returns due schedule tasks.+func (s *SyncTaskService) ListDueTasks(ctx context.Context, now time.Time) ([]dao.SyncTask, error) {+ tasks, err := s.taskDAO.ListDueTasks(ctx, now, 128)+ if err != nil {+ return nil, err+ }+ out := make([]dao.SyncTask, 0, len(tasks))+ for _, task := range tasks {+ out = append(out, dao.SyncTask{SyncLogs: task})+ }+ return out, nil+}++// Claim marks a scheduled task running if no other scanner claimed it first.+func (s *SyncTaskService) Claim(ctx context.Context, taskID string) (bool, error) {+ claimed, err := s.taskDAO.ClaimTask(ctx, taskID, time.Now().Local())+ if err != nil || !claimed {+ return claimed, err+ }++ taskContext, err := s.taskDAO.GetTaskContext(ctx, taskID)+ if err != nil {+ return true, err+ }++ return true, s.taskDAO.MarkConnectorRunning(ctx, taskContext.Connector.ID)+}++// GetContext loads a task execution context.+func (s *SyncTaskService) GetContext(ctx context.Context, taskID string) (SyncTaskContext, error) {+ return s.taskDAO.GetTaskContext(ctx, taskID)+}++// RescheduleClaimed puts a claimed task back into schedule state.+func (s *SyncTaskService) RescheduleClaimed(ctx context.Context, taskID string) error {+ return s.taskDAO.RescheduleClaimed(ctx, taskID)+}++// Fail records a failed task.+func (s *SyncTaskService) Fail(ctx context.Context, taskID, connectorID string, err error) error {+ message := ""+ if err != nil {+ message = err.Error()+ }+ return s.taskDAO.FailTask(ctx, taskID, connectorID, message, 1)+}++// CompleteSync commits a successful SYNC task and schedules the next one.+func (s *SyncTaskService) CompleteSync(ctx context.Context, taskContext SyncTaskContext, pollRangeEnd time.Time, stats SyncStats) error {+ changed := stats.Added + stats.Updated+ return s.taskDAO.CompleteSyncTask(ctx, taskContext, pollRangeEnd, changed, changed, stats.ErrorCount, stats.ErrorMsg)+}++// CompletePrune commits a successful PRUNE task and schedules the next one.+func (s *SyncTaskService) CompletePrune(ctx context.Context, taskContext SyncTaskContext, removed int64) error {+ return s.taskDAO.CompletePruneTask(ctx, taskContext, removed)+}++// RecoverStaleRunning restores timed-out running tasks.+func (s *SyncTaskService) RecoverStaleRunning(ctx context.Context, now time.Time) error {+ _, err := s.taskDAO.RecoverStaleRunning(ctx, now)+ return err+}++// IsFromBeginning reports whether a task is a full sync.+func IsFromBeginning(value *string) bool {+ if value == nil {+ return false+ }+ return strings.TrimSpace(*value) == "1"+}++// SourceType returns document source_type.+func SourceType(source, connectorID string) string {+ return fmt.Sprintf("%s/%s", source, connectorID)+}diff --git a/internal/syncer/config.go b/internal/syncer/config.gonew file mode 100644index 0000000000..4abd51b219--- /dev/null+++ b/internal/syncer/config.go@@ -0,0 +1,79 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "time"+)++// Config contains runtime limits for the datasource syncer.+type Config struct {+ PollInterval time.Duration+ TaskConcurrency int+ TaskQueueSize int+ PerTaskItemConcurrency int+ GlobalItemConcurrency int+ ItemRetryCount int+ ItemRetryBaseDelay time.Duration+}++// DefaultConfig returns the first-version syncer defaults.+func DefaultConfig() Config {+ return Config{+ PollInterval: 3 * time.Second,+ TaskConcurrency: 3,+ TaskQueueSize: 32,+ PerTaskItemConcurrency: 4,+ GlobalItemConcurrency: 12,+ ItemRetryCount: 3,+ ItemRetryBaseDelay: time.Second,+ }+}++// Normalize fills unset or invalid config values with defaults.+func (c Config) Normalize() Config {+ def := DefaultConfig()+ if c.PollInterval <= 0 {+ c.PollInterval = def.PollInterval+ }++ if c.TaskConcurrency <= 0 {+ c.TaskConcurrency = def.TaskConcurrency+ }++ if c.TaskQueueSize <= 0 {+ c.TaskQueueSize = def.TaskQueueSize+ }++ if c.PerTaskItemConcurrency <= 0 {+ c.PerTaskItemConcurrency = def.PerTaskItemConcurrency+ }++ if c.GlobalItemConcurrency <= 0 {+ c.GlobalItemConcurrency = def.GlobalItemConcurrency+ }++ if c.ItemRetryCount <= 0 {+ c.ItemRetryCount = def.ItemRetryCount+ }++ if c.ItemRetryBaseDelay <= 0 {+ c.ItemRetryBaseDelay = def.ItemRetryBaseDelay+ }++ return c+}diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.gonew file mode 100644index 0000000000..0c0ab8bfe8--- /dev/null+++ b/internal/syncer/connector/github.go@@ -0,0 +1,691 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "os"+ "strconv"+ "strings"+ "time"++ "ragflow/internal/utility"+)++const (+ defaultGitHubBatchSize = 32+ githubItemsPerPage = 100+ githubRequestTimeout = 60 * time.Second+)++// GitHubConnector reads GitHub issues and pull requests.+type GitHubConnector struct {+ owner string+ repos []string+ includePRs bool+ includeIssues bool+ token string+ batchSize int+ baseURL string+ doJSON func(ctx context.Context, apiURL string, out any) (http.Header, error)+}++// NewGitHubConnector creates a GitHub connector from Python-compatible config.+func NewGitHubConnector(config map[string]any) (*GitHubConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ token, _ := credentials["github_access_token"].(string)+ baseURL := strings.TrimRight(os.Getenv("GITHUB_CONNECTOR_BASE_URL"), "/")+ if baseURL == "" {+ baseURL = "https://api.github.com"+ }+ return &GitHubConnector{+ owner: strings.TrimSpace(stringConfig(config["repository_owner"])),+ repos: splitGitHubRepos(stringConfig(config["repository_name"])),+ includePRs: configBoolDefault(config["include_pull_requests"], true),+ includeIssues: configBoolDefault(config["include_issues"], true),+ token: strings.TrimSpace(token),+ batchSize: configInt(config["batch_size"], defaultGitHubBatchSize),+ baseURL: baseURL,+ }, nil+}++// Validate validates GitHub connector settings and credentials.+func (c *GitHubConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("github connector is nil")+ }+ if c.owner == "" {+ return fmt.Errorf("Invalid connector settings: 'repo_owner' must be provided")+ }+ if c.token == "" {+ return fmt.Errorf("Missing github_access_token in credentials")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ repos, err := c.listRepos(ctx)+ if err != nil {+ return err+ }+ if len(repos) == 0 {+ return fmt.Errorf("found no repos for GitHub owner %s", c.owner)+ }+ return nil+}++// OpenSync opens one GitHub sync session.+func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ repos, err := c.listRepos(ctx)+ if err != nil {+ return nil, err+ }+ return &githubSyncSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1, windowStart: request.WindowStart, windowEnd: request.WindowEnd}, nil+}++// OpenPrune opens one complete GitHub prune snapshot session.+func (c *GitHubConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ repos, err := c.listRepos(ctx)+ if err != nil {+ return nil, err+ }+ return &githubPruneSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1}, nil+}++// listRepos returns configured repositories or all owner repositories.+func (c *GitHubConnector) listRepos(ctx context.Context) ([]githubRepo, error) {+ if len(c.repos) > 0 {+ repos := make([]githubRepo, 0, len(c.repos))+ for _, repoName := range c.repos {+ var repo githubRepo+ if _, err := c.getJSON(ctx, c.apiURL("/repos/"+url.PathEscape(c.owner)+"/"+url.PathEscape(repoName), nil), &repo); err != nil {+ return nil, err+ }+ repos = append(repos, repo)+ }+ return repos, nil+ }++ repos, err := c.listRepoEndpoint(ctx, "/orgs/"+url.PathEscape(c.owner)+"/repos")+ if err == nil {+ return repos, nil+ }+ return c.listRepoEndpoint(ctx, "/users/"+url.PathEscape(c.owner)+"/repos")+}++// listRepoEndpoint returns all repos from an org or user endpoint.+func (c *GitHubConnector) listRepoEndpoint(ctx context.Context, path string) ([]githubRepo, error) {+ repos := []githubRepo{}+ for page := 1; ; page++ {+ query := url.Values{"per_page": {strconv.Itoa(githubItemsPerPage)}, "page": {strconv.Itoa(page)}}+ var batch []githubRepo+ headers, err := c.getJSON(ctx, c.apiURL(path, query), &batch)+ if err != nil {+ return nil, err+ }+ repos = append(repos, batch...)+ if !hasNextPage(headers) || len(batch) == 0 {+ break+ }+ }+ return repos, nil+}++// listPullRequestPage returns one page of GitHub pull requests.+func (c *GitHubConnector) listPullRequestPage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]SourceDocument, bool, error) {+ var batch []githubPullRequest+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/pulls", githubListQuery(page, pageSize)), &batch)+ if err != nil {+ return nil, false, err+ }+ documents := make([]SourceDocument, 0, len(batch))+ doneByWindow := false+ for _, pr := range batch {+ if beforeOrAtWindowStart(pr.UpdatedAt, windowStart) {+ doneByWindow = true+ break+ }+ if afterWindowEnd(pr.UpdatedAt, windowEnd) {+ continue+ }+ documents = append(documents, pr.toSourceDocument(fullName))+ }+ done := doneByWindow || !hasNextPage(headers) || len(batch) == 0+ return documents, done, nil+}++// listIssuePage returns one page of GitHub issues.+func (c *GitHubConnector) listIssuePage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]SourceDocument, bool, error) {+ var batch []githubIssue+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/issues", githubListQuery(page, pageSize)), &batch)+ if err != nil {+ return nil, false, err+ }+ documents := make([]SourceDocument, 0, len(batch))+ doneByWindow := false+ for _, issue := range batch {+ if issue.PullRequest != nil {+ continue+ }+ if beforeOrAtWindowStart(issue.UpdatedAt, windowStart) {+ doneByWindow = true+ break+ }+ if afterWindowEnd(issue.UpdatedAt, windowEnd) {+ continue+ }+ documents = append(documents, issue.toSourceDocument(fullName))+ }+ done := doneByWindow || !hasNextPage(headers) || len(batch) == 0+ return documents, done, nil+}++// listPullRequestSlimPage returns one page of GitHub pull request IDs.+func (c *GitHubConnector) listPullRequestSlimPage(ctx context.Context, fullName string, page, pageSize int) ([]SlimDocument, bool, error) {+ var batch []githubPullRequest+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/pulls", githubListQuery(page, pageSize)), &batch)+ if err != nil {+ return nil, false, err+ }+ documents := make([]SlimDocument, 0, len(batch))+ for _, pr := range batch {+ documents = append(documents, SlimDocument{SourceID: pr.HTMLURL})+ }+ return documents, !hasNextPage(headers) || len(batch) == 0, nil+}++// listIssueSlimPage returns one page of GitHub issue IDs.+func (c *GitHubConnector) listIssueSlimPage(ctx context.Context, fullName string, page, pageSize int) ([]SlimDocument, bool, error) {+ var batch []githubIssue+ headers, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/issues", githubListQuery(page, pageSize)), &batch)+ if err != nil {+ return nil, false, err+ }+ documents := make([]SlimDocument, 0, len(batch))+ for _, issue := range batch {+ if issue.PullRequest != nil {+ continue+ }+ documents = append(documents, SlimDocument{SourceID: issue.HTMLURL})+ }+ return documents, !hasNextPage(headers) || len(batch) == 0, nil+}++// githubListQuery builds standard GitHub list query parameters.+func githubListQuery(page, pageSize int) url.Values {+ if pageSize <= 0 {+ pageSize = defaultGitHubBatchSize+ }+ return url.Values{+ "state": {"all"},+ "sort": {"updated"},+ "direction": {"desc"},+ "per_page": {strconv.Itoa(pageSize)},+ "page": {strconv.Itoa(page)},+ }+}++// getJSON fetches a GitHub API response into out.+func (c *GitHubConnector) getJSON(ctx context.Context, apiURL string, out any) (http.Header, error) {+ if c.doJSON != nil {+ return c.doJSON(ctx, apiURL, out)+ }+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)+ if err != nil {+ return nil, err+ }+ client := utility.PinnedHTTPClient(hostname, resolvedIP, githubRequestTimeout)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Accept", "application/vnd.github+json")+ req.Header.Set("X-GitHub-Api-Version", "2022-11-28")+ req.Header.Set("Authorization", "Bearer "+c.token)+ resp, err := client.Do(req)+ if err != nil {+ return nil, fmt.Errorf("failed to fetch GitHub API: %w", err)+ }+ defer resp.Body.Close()+ if resp.StatusCode >= 400 {+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return nil, fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))+ }+ if err = json.NewDecoder(resp.Body).Decode(out); err != nil {+ return nil, err+ }+ return resp.Header.Clone(), nil+}++// apiURL builds a GitHub API URL.+func (c *GitHubConnector) apiURL(path string, query url.Values) string {+ path = strings.TrimLeft(path, "/")+ u := strings.TrimRight(c.baseURL, "/") + "/" + path+ if len(query) == 0 {+ return u+ }+ return u + "?" + query.Encode()+}++type githubSyncSession struct {+ connector *GitHubConnector+ repos []githubRepo+ repoIndex int+ stage string+ page int+ batchSize int+ windowStart *time.Time+ windowEnd time.Time+ buffer []SourceDocument+}++// NextBatch returns the next GitHub document batch.+func (s *githubSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ if s.repoIndex >= len(s.repos) {+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ batch, err := s.nextDocumentPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ remaining := s.batchSize - len(documents)+ if len(batch) > remaining {+ documents = append(documents, batch[:remaining]...)+ s.buffer = append(s.buffer, batch[remaining:]...)+ break+ }+ documents = append(documents, batch...)+ }+ return SyncBatch{Documents: documents}, nil+}++// Close closes the GitHub sync session.+func (s *githubSyncSession) Close() error {+ return nil+}++type githubPruneSession struct {+ connector *GitHubConnector+ repos []githubRepo+ repoIndex int+ stage string+ page int+ batchSize int+ buffer []SlimDocument+}++// NextBatch returns the next GitHub prune snapshot batch.+func (s *githubPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ if s.repoIndex >= len(s.repos) {+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ break+ }+ batch, err := s.nextSlimPage(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ remaining := s.batchSize - len(documents)+ if len(batch) > remaining {+ documents = append(documents, batch[:remaining]...)+ s.buffer = append(s.buffer, batch[remaining:]...)+ break+ }+ documents = append(documents, batch...)+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the GitHub prune session.+func (s *githubPruneSession) Close() error {+ return nil+}++const (+ githubStagePRs = "prs"+ githubStageIssues = "issues"+)++// nextDocumentPage fetches one GitHub API page for sync.+func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {+ repo := s.repos[s.repoIndex]+ switch s.stage {+ case githubStagePRs:+ if !s.connector.includePRs {+ s.advanceStage()+ return nil, nil+ }+ docs, done, err := s.connector.listPullRequestPage(ctx, repo.FullName, s.page, s.batchSize, s.windowStart, s.windowEnd)+ if err != nil {+ return nil, err+ }+ if done {+ s.advanceStage()+ } else {+ s.page+++ }+ return docs, nil+ case githubStageIssues:+ if !s.connector.includeIssues {+ s.advanceRepo()+ return nil, nil+ }+ docs, done, err := s.connector.listIssuePage(ctx, repo.FullName, s.page, s.batchSize, s.windowStart, s.windowEnd)+ if err != nil {+ return nil, err+ }+ if done {+ s.advanceRepo()+ } else {+ s.page+++ }+ return docs, nil+ default:+ s.advanceRepo()+ return nil, nil+ }+}++// nextSlimPage fetches one GitHub API page for prune.+func (s *githubPruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {+ repo := s.repos[s.repoIndex]+ switch s.stage {+ case githubStagePRs:+ if !s.connector.includePRs {+ s.advanceStage()+ return nil, nil+ }+ docs, done, err := s.connector.listPullRequestSlimPage(ctx, repo.FullName, s.page, s.batchSize)+ if err != nil {+ return nil, err+ }+ if done {+ s.advanceStage()+ } else {+ s.page+++ }+ return docs, nil+ case githubStageIssues:+ if !s.connector.includeIssues {+ s.advanceRepo()+ return nil, nil+ }+ docs, done, err := s.connector.listIssueSlimPage(ctx, repo.FullName, s.page, s.batchSize)+ if err != nil {+ return nil, err+ }+ if done {+ s.advanceRepo()+ } else {+ s.page+++ }+ return docs, nil+ default:+ s.advanceRepo()+ return nil, nil+ }+}++// advanceStage moves a GitHub session from PRs to issues.+func (s *githubSyncSession) advanceStage() {+ s.stage = githubStageIssues+ s.page = 1+}++// advanceRepo moves a GitHub session to the next repository.+func (s *githubSyncSession) advanceRepo() {+ s.repoIndex+++ s.stage = githubStagePRs+ s.page = 1+}++// advanceStage moves a GitHub prune session from PRs to issues.+func (s *githubPruneSession) advanceStage() {+ s.stage = githubStageIssues+ s.page = 1+}++// advanceRepo moves a GitHub prune session to the next repository.+func (s *githubPruneSession) advanceRepo() {+ s.repoIndex+++ s.stage = githubStagePRs+ s.page = 1+}++type githubRepo struct {+ FullName string `json:"full_name"`+}++type githubPullRequest struct {+ HTMLURL string `json:"html_url"`+ Number int `json:"number"`+ Title string `json:"title"`+ Body string `json:"body"`+ State string `json:"state"`+ MergedAt *time.Time `json:"merged_at"`+ UpdatedAt time.Time `json:"updated_at"`+ CreatedAt *time.Time `json:"created_at"`+ ClosedAt *time.Time `json:"closed_at"`+ User *githubUser `json:"user"`+ Assignees []githubUser `json:"assignees"`+ Labels []githubLabel `json:"labels"`+}++// toSourceDocument converts a pull request into the syncer model.+func (p githubPullRequest) toSourceDocument(repo string) SourceDocument {+ body := []byte(p.Body)+ return SourceDocument{+ SourceID: p.HTMLURL,+ SemanticIdentifier: fmt.Sprintf("%d:%s", p.Number, sanitizeGitHubName(p.Title, "md")),+ Extension: ".md",+ Blob: body,+ UpdatedAt: p.UpdatedAt.UTC(),+ SizeBytes: int64(len(body)),+ Metadata: map[string]any{+ "object_type": "PullRequest",+ "id": strconv.Itoa(p.Number),+ "state": p.State,+ "repo": repo,+ "merged": strconv.FormatBool(p.MergedAt != nil),+ "labels": githubLabelNames(p.Labels),+ "user": p.User.metadata(),+ "assignees": githubUsersMetadata(p.Assignees),+ },+ }+}++type githubIssue struct {+ HTMLURL string `json:"html_url"`+ Number int `json:"number"`+ Title string `json:"title"`+ Body string `json:"body"`+ State string `json:"state"`+ UpdatedAt time.Time `json:"updated_at"`+ CreatedAt *time.Time `json:"created_at"`+ ClosedAt *time.Time `json:"closed_at"`+ User *githubUser `json:"user"`+ Assignees []githubUser `json:"assignees"`+ Labels []githubLabel `json:"labels"`+ PullRequest *struct{} `json:"pull_request"`+}++// toSourceDocument converts an issue into the syncer model.+func (i githubIssue) toSourceDocument(repo string) SourceDocument {+ body := []byte(i.Body)+ return SourceDocument{+ SourceID: i.HTMLURL,+ SemanticIdentifier: fmt.Sprintf("%d:%s", i.Number, sanitizeGitHubName(i.Title, "md")),+ Extension: ".md",+ Blob: body,+ UpdatedAt: i.UpdatedAt.UTC(),+ SizeBytes: int64(len(body)),+ Metadata: map[string]any{+ "object_type": "Issue",+ "id": strconv.Itoa(i.Number),+ "state": i.State,+ "repo": repo,+ "labels": githubLabelNames(i.Labels),+ "user": i.User.metadata(),+ "assignees": githubUsersMetadata(i.Assignees),+ },+ }+}++type githubUser struct {+ Login string `json:"login"`+ Name string `json:"name"`+ Email string `json:"email"`+}++// metadata returns Python-like user metadata.+func (u *githubUser) metadata() map[string]string {+ if u == nil {+ return nil+ }+ out := map[string]string{}+ if u.Login != "" {+ out["login"] = u.Login+ }+ if u.Name != "" {+ out["name"] = u.Name+ }+ if u.Email != "" {+ out["email"] = u.Email+ }+ return out+}++type githubLabel struct {+ Name string `json:"name"`+}++// githubLabelNames returns label names.+func githubLabelNames(labels []githubLabel) []string {+ out := make([]string, 0, len(labels))+ for _, label := range labels {+ if label.Name != "" {+ out = append(out, label.Name)+ }+ }+ return out+}++// githubUsersMetadata returns metadata for users.+func githubUsersMetadata(users []githubUser) []map[string]string {+ out := make([]map[string]string, 0, len(users))+ for i := range users {+ out = append(out, (&users[i]).metadata())+ }+ return out+}++// hasNextPage reports whether a GitHub Link header has rel next.+func hasNextPage(headers http.Header) bool {+ return strings.Contains(headers.Get("Link"), `rel="next"`)+}++// beforeOrAtWindowStart reports whether an updated time is outside the lower bound.+func beforeOrAtWindowStart(updatedAt time.Time, windowStart *time.Time) bool {+ return windowStart != nil && !updatedAt.After(*windowStart)+}++// afterWindowEnd reports whether an updated time is outside the upper bound.+func afterWindowEnd(updatedAt time.Time, windowEnd time.Time) bool {+ return !windowEnd.IsZero() && updatedAt.After(windowEnd)+}++// splitGitHubRepos parses Python's comma-separated repository_name config.+func splitGitHubRepos(value string) []string {+ parts := strings.Split(value, ",")+ repos := make([]string, 0, len(parts))+ for _, part := range parts {+ part = strings.TrimSpace(part)+ if part != "" {+ repos = append(repos, part)+ }+ }+ return repos+}++// stringConfig reads a string config value.+func stringConfig(value any) string {+ if value == nil {+ return ""+ }+ return fmt.Sprint(value)+}++// configBoolDefault reads a Python JSON bool/string flag with a default.+func configBoolDefault(value any, fallback bool) bool {+ if value == nil {+ return fallback+ }+ switch typed := value.(type) {+ case bool:+ return typed+ case string:+ switch strings.ToLower(strings.TrimSpace(typed)) {+ case "1", "true", "yes", "on":+ return true+ case "0", "false", "no", "off":+ return false+ }+ }+ return fallback+}++// sanitizeGitHubName mirrors Python's sanitized markdown filename intent.+func sanitizeGitHubName(name, extension string) string {+ name = strings.TrimSpace(name)+ if name == "" {+ name = "github"+ }+ replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", `"`, "_", "<", "_", ">", "_", "|", "_")+ name = replacer.Replace(name)+ if !strings.HasSuffix(strings.ToLower(name), "."+extension) {+ name += "." + extension+ }+ return name+}diff --git a/internal/syncer/connector/github_test.go b/internal/syncer/connector/github_test.gonew file mode 100644index 0000000000..7ceae7b19f--- /dev/null+++ b/internal/syncer/connector/github_test.go@@ -0,0 +1,101 @@+package connector++import (+ "context"+ "encoding/json"+ "net/http"+ "net/url"+ "testing"+)++// TestGitHubConnectorOpenPrune verifies PRUNE returns Python-compatible html_url IDs.+func TestGitHubConnectorOpenPrune(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "ragflow",+ "include_pull_requests": true,+ "include_issues": true,+ "batch_size": 10,+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubFixtureDoJSON(t)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ got := []string{}+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ want := []string{+ "https://github.com/openai/ragflow/pull/7",+ "https://github.com/openai/ragflow/issues/3",+ }+ if len(got) != len(want) {+ t.Fatalf("ids len = %d, want %d: %v", len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("ids[%d] = %q, want %q", i, got[i], want[i])+ }+ }+}++// githubFixtureDoJSON returns a fixture GitHub JSON transport.+func githubFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ t.Helper()+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ fixtures := map[string]string{+ "/repos/openai/ragflow": `{"full_name":"openai/ragflow"}`,+ "/repos/openai/ragflow/pulls": `[{+ "html_url":"https://github.com/openai/ragflow/pull/7",+ "number":7,+ "title":"Add syncer",+ "body":"PR body",+ "state":"open",+ "updated_at":"2026-01-03T00:00:00Z",+ "labels":[{"name":"sync"}],+ "user":{"login":"alice"}+ }]`,+ "/repos/openai/ragflow/issues": `[{+ "html_url":"https://github.com/openai/ragflow/issues/3",+ "number":3,+ "title":"Prune bug",+ "body":"Issue body",+ "state":"open",+ "updated_at":"2026-01-02T00:00:00Z",+ "labels":[{"name":"bug"}],+ "user":{"login":"bob"}+ },{+ "html_url":"https://github.com/openai/ragflow/pull/7",+ "number":7,+ "title":"PR shadow",+ "body":"skip me",+ "state":"open",+ "updated_at":"2026-01-03T00:00:00Z",+ "pull_request":{}+ }]`,+ }+ body, ok := fixtures[parsed.Path]+ if !ok {+ t.Fatalf("unexpected api path %s", parsed.Path)+ }+ if err = json.Unmarshal([]byte(body), out); err != nil {+ t.Fatalf("decode fixture: %v", err)+ }+ return http.Header{}, nil+ }+}diff --git a/internal/syncer/connector/interface.go b/internal/syncer/connector/interface.gonew file mode 100644index 0000000000..c63f145f1b--- /dev/null+++ b/internal/syncer/connector/interface.go@@ -0,0 +1,51 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import "context"++// SyncSession streams source documents for one fixed sync window.+type SyncSession interface {+ // NextBatch returns the next source document batch.+ NextBatch(ctx context.Context) (SyncBatch, error)+ // Close releases connector resources.+ Close() error+}++// Fetcher lazily downloads a source document body.+type Fetcher interface {+ // Fetch downloads a delayed source document body.+ Fetch(ctx context.Context, ref FetchReference) ([]byte, error)+}++// PruneSession streams a complete slim source snapshot.+type PruneSession interface {+ // NextBatch returns the next slim snapshot batch.+ NextBatch(ctx context.Context) (PruneBatch, error)+ // Close releases connector resources.+ Close() error+}++// Connector validates configuration and opens sync/prune sessions.+type Connector interface {+ // Validate validates connector configuration and credentials.+ Validate(ctx context.Context) error+ // OpenSync opens one synchronization session.+ OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error)+ // OpenPrune opens one complete prune snapshot session.+ OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error)+}diff --git a/internal/syncer/connector/mock/mock.go b/internal/syncer/connector/mock/mock.gonew file mode 100644index 0000000000..7a1e3c8bad--- /dev/null+++ b/internal/syncer/connector/mock/mock.go@@ -0,0 +1,109 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package mock++import (+ "context"+ "io"+ syncerconnector "ragflow/internal/syncer/connector"+)++// Connector is a programmable connector for syncer tests.+type Connector struct {+ ValidateErr error+ SyncBatches []syncerconnector.SyncBatch+ SyncErrAt int+ PruneBatches []syncerconnector.PruneBatch+ PruneErrAt int+ FetchBlobs map[string][]byte+ OnSyncBatch func(index int)+ OnPruneBatch func(index int)+}++// Validate returns the configured validation error.+func (c *Connector) Validate(ctx context.Context) error {+ return c.ValidateErr+}++// OpenSync opens a mock sync session.+func (c *Connector) OpenSync(ctx context.Context, request syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {+ return &SyncSession{connector: c}, nil+}++// OpenPrune opens a mock prune session.+func (c *Connector) OpenPrune(ctx context.Context, request syncerconnector.PruneRequest) (syncerconnector.PruneSession, error) {+ return &PruneSession{connector: c}, nil+}++// SyncSession streams configured sync batches.+type SyncSession struct {+ connector *Connector+ index int+}++// NextBatch returns the next configured sync batch.+func (s *SyncSession) NextBatch(ctx context.Context) (syncerconnector.SyncBatch, error) {+ if s.connector.SyncErrAt > 0 && s.index+1 == s.connector.SyncErrAt {+ return syncerconnector.SyncBatch{}, io.ErrUnexpectedEOF+ }+ if s.index >= len(s.connector.SyncBatches) {+ return syncerconnector.SyncBatch{}, io.EOF+ }+ if s.connector.OnSyncBatch != nil {+ s.connector.OnSyncBatch(s.index)+ }+ batch := s.connector.SyncBatches[s.index]+ s.index+++ return batch, nil+}++// Fetch returns a configured lazy blob.+func (s *SyncSession) Fetch(ctx context.Context, ref syncerconnector.FetchReference) ([]byte, error) {+ return s.connector.FetchBlobs[ref.Key], nil+}++// Close closes the mock sync session.+func (s *SyncSession) Close() error {+ return nil+}++// PruneSession streams configured prune batches.+type PruneSession struct {+ connector *Connector+ index int+}++// NextBatch returns the next configured prune batch.+func (s *PruneSession) NextBatch(ctx context.Context) (syncerconnector.PruneBatch, error) {+ if s.connector.PruneErrAt > 0 && s.index+1 == s.connector.PruneErrAt {+ return syncerconnector.PruneBatch{}, io.ErrUnexpectedEOF+ }+ if s.index >= len(s.connector.PruneBatches) {+ return syncerconnector.PruneBatch{}, io.EOF+ }+ if s.connector.OnPruneBatch != nil {+ s.connector.OnPruneBatch(s.index)+ }+ batch := s.connector.PruneBatches[s.index]+ s.index+++ return batch, nil+}++// Close closes the mock prune session.+func (s *PruneSession) Close() error {+ return nil+}diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.gonew file mode 100644index 0000000000..4454b0f866--- /dev/null+++ b/internal/syncer/connector/models.go@@ -0,0 +1,70 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import "time"++// SourceDocument is the normalized document emitted by datasource connectors.+type SourceDocument struct {+ SourceID string+ SemanticIdentifier string+ Extension string+ Blob []byte+ FetchRef *FetchReference+ UpdatedAt time.Time+ SizeBytes int64+ Metadata map[string]any+ Fingerprint string+}++// FetchReference describes a lazy document fetch.+type FetchReference struct {+ Key string+ SizeHint int64+}++// SyncRequest describes one fixed sync window.+type SyncRequest struct {+ TaskID string+ ConnectorID string+ KBID string+ FromBeginning bool+ WindowStart *time.Time+ WindowEnd time.Time+}++// SyncBatch contains one serially processed batch.+type SyncBatch struct {+ Documents []SourceDocument+}++// PruneRequest describes one complete prune snapshot request.+type PruneRequest struct {+ TaskID string+ ConnectorID string+ KBID string+}++// SlimDocument is the minimal prune snapshot row.+type SlimDocument struct {+ SourceID string+}++// PruneBatch contains one slim snapshot batch.+type PruneBatch struct {+ Documents []SlimDocument+}diff --git a/internal/syncer/connector/registry.go b/internal/syncer/connector/registry.gonew file mode 100644index 0000000000..6608fcc003--- /dev/null+++ b/internal/syncer/connector/registry.go@@ -0,0 +1,76 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "fmt"+ "sync"+)++// Factory creates a connector for a task context.+type Factory func(ctx context.Context, taskContext any) (Connector, error)++// Registry maps Python connector source names to Go connector factories.+type Registry struct {+ mu sync.RWMutex+ factories map[string]Factory+}++// NewRegistry creates an empty connector registry.+func NewRegistry() *Registry {+ return &Registry{factories: map[string]Factory{}}+}++// Register adds or replaces a connector factory.+func (r *Registry) Register(source string, factory Factory) {+ r.mu.Lock()+ defer r.mu.Unlock()+ r.factories[source] = factory+}++// Open creates a connector for a task context.+func (r *Registry) Open(ctx context.Context, taskContext any) (Connector, error) {+ row, ok := taskContext.(interface{ ConnectorSource() string })+ if ok {+ return r.openSource(ctx, row.ConnectorSource(), taskContext)+ }+ sourceProvider, ok := taskContext.(interface{ Source() string })+ if ok {+ return r.openSource(ctx, sourceProvider.Source(), taskContext)+ }+ rowContext, ok := taskContext.(struct{ Connector struct{ Source string } })+ if ok {+ return r.openSource(ctx, rowContext.Connector.Source, taskContext)+ }+ source := ""+ if value, ok := any(taskContext).(interface{ GetSource() string }); ok {+ source = value.GetSource()+ }+ return r.openSource(ctx, source, taskContext)+}++// openSource creates a connector for a known source.+func (r *Registry) openSource(ctx context.Context, source string, taskContext any) (Connector, error) {+ r.mu.RLock()+ factory := r.factories[source]+ r.mu.RUnlock()+ if factory == nil {+ return nil, fmt.Errorf("unsupported connector source %q", source)+ }+ return factory(ctx, taskContext)+}diff --git a/internal/syncer/connector/rss.go b/internal/syncer/connector/rss.gonew file mode 100644index 0000000000..dc626206e0--- /dev/null+++ b/internal/syncer/connector/rss.go@@ -0,0 +1,538 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "bytes"+ "context"+ "crypto/md5"+ "encoding/hex"+ "encoding/xml"+ "fmt"+ "html"+ "io"+ "net/http"+ "net/url"+ "regexp"+ "strconv"+ "strings"+ "time"++ "ragflow/internal/utility"++ "github.com/zeebo/xxh3"+)++const (+ defaultRSSBatchSize = 32+ maxRSSFeedSize = 32 * 1024 * 1024+ rssFetchTimeout = 60 * time.Second+ rssFetchRetryCount = 3+)++var (+ rssHTMLScriptStyleRE = regexp.MustCompile(`(?is)<(script|style)[^>]*>.*?</(script|style)>`)+ rssHTMLTagRE = regexp.MustCompile(`(?s)<[^>]+>`)+ rssWhitespaceRE = regexp.MustCompile(`[ \t]+`)+ rssNewlineRE = regexp.MustCompile(`\n{3,}`)+)++// RSSConnector reads RSS and Atom feeds.+type RSSConnector struct {+ feedURL string+ batchSize int+ fetchFeed func(ctx context.Context, feedURL string) ([]byte, error)+ entries []rssEntry+}++// NewRSSConnector creates an RSS connector from Python-compatible config.+func NewRSSConnector(config map[string]any) (*RSSConnector, error) {+ feedURL, _ := config["feed_url"].(string)+ batchSize := configInt(config["batch_size"], defaultRSSBatchSize)+ return &RSSConnector{feedURL: strings.TrimSpace(feedURL), batchSize: batchSize}, nil+}++// Validate validates RSS connector settings and feed readability.+func (c *RSSConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("rss connector is nil")+ }+ if c.feedURL == "" {+ return fmt.Errorf("RSS feed URL is required")+ }++ parsed, err := url.Parse(c.feedURL)+ if err != nil || parsed.Scheme == "" || parsed.Host == "" {+ return fmt.Errorf("invalid RSS feed URL")+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return fmt.Errorf("RSS feed URL must use HTTP or HTTPS")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }++ entries, err := c.loadEntries(ctx)+ if err != nil {+ return err+ }+ if len(entries) == 0 {+ return fmt.Errorf("RSS feed contains no entries")+ }++ c.entries = entries+ return nil+}++// OpenSync opens one RSS sync session.+func (c *RSSConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ entries, err := c.loadEntries(ctx)+ if err != nil {+ return nil, err+ }++ documents := make([]SourceDocument, 0, len(entries))+ for _, entry := range entries {+ if !request.WindowEnd.IsZero() && entry.updatedAt.After(request.WindowEnd) {+ continue+ }+ if !request.FromBeginning && request.WindowStart != nil { // not reindex+ if !entry.updatedAt.After(*request.WindowStart) {+ continue+ }+ }+ documents = append(documents, entry.toSourceDocument(c.feedURL))+ }+ return &rssSyncSession{documents: documents, batchSize: c.batchSize}, nil+}++// OpenPrune opens one complete RSS prune snapshot session.+func (c *RSSConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ entries, err := c.loadEntries(ctx)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(entries))+ for _, entry := range entries {+ documents = append(documents, SlimDocument{SourceID: entry.sourceID()})+ }+ return &rssPruneSession{documents: documents, batchSize: c.batchSize}, nil+}++// loadEntries fetches and parses the configured feed.+func (c *RSSConnector) loadEntries(ctx context.Context) ([]rssEntry, error) {+ if c.entries != nil {+ return c.entries, nil+ }+ if c.fetchFeed != nil {+ data, err := c.fetchFeed(ctx, c.feedURL)+ if err != nil {+ return nil, err+ }+ return parseRSSFeed(data)+ }+ data, _, _, err := fetchRSSFeedWithRetry(ctx, c.feedURL)+ if err != nil {+ return nil, err+ }+ entries, err := parseRSSFeed(data)+ if err != nil {+ return nil, err+ }+ return entries, nil+}++// fetchRSSFeedWithRetry fetches a feed with bounded retries for transient slowness.+func fetchRSSFeedWithRetry(ctx context.Context, feedURL string) ([]byte, http.Header, string, error) {+ var lastErr error+ for attempt := 1; attempt <= rssFetchRetryCount; attempt++ {+ data, headers, finalURL, err := utility.FetchRemoteFileSafelyWithTimeout(ctx, feedURL, maxRSSFeedSize, rssFetchTimeout)+ if err == nil {+ return data, headers, finalURL, nil+ }+ lastErr = err+ if attempt == rssFetchRetryCount {+ break+ }+ select {+ case <-ctx.Done():+ return nil, nil, "", ctx.Err()+ case <-time.After(time.Duration(attempt) * time.Second):+ }+ }+ return nil, nil, "", lastErr+}++type rssSyncSession struct {+ documents []SourceDocument+ batchSize int+ index int+}++// NextBatch returns the next RSS document batch.+func (s *rssSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ if s.index >= len(s.documents) {+ return SyncBatch{}, io.EOF+ }+ end := s.index + s.batchSize+ if end > len(s.documents) {+ end = len(s.documents)+ }+ batch := SyncBatch{Documents: s.documents[s.index:end]}+ s.index = end+ return batch, nil+}++// Close closes the RSS sync session.+func (s *rssSyncSession) Close() error {+ return nil+}++type rssPruneSession struct {+ documents []SlimDocument+ batchSize int+ index int+}++// NextBatch returns the next RSS prune snapshot batch.+func (s *rssPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.index >= len(s.documents) {+ return PruneBatch{}, io.EOF+ }+ end := s.index + s.batchSize+ if end > len(s.documents) {+ end = len(s.documents)+ }+ batch := PruneBatch{Documents: s.documents[s.index:end]}+ s.index = end+ return batch, nil+}++// Close closes the RSS prune session.+func (s *rssPruneSession) Close() error {+ return nil+}++type rssEntry struct {+ id string+ link string+ title string+ author string+ categories []string+ content string+ summary string+ updatedAt time.Time+}++// sourceID returns the RSS source document ID.+func (e rssEntry) sourceID() string {+ stable := firstNonEmpty(e.id, e.link, e.title)+ sum := md5.Sum([]byte(stable))+ return "rss:" + hex.EncodeToString(sum[:])+}++// semanticIdentifier returns the human-readable document name seed.+func (e rssEntry) semanticIdentifier() string {+ return firstNonEmpty(e.title, e.link, e.id, "rss-entry")+}++// toSourceDocument converts one feed entry into the syncer model.+func (e rssEntry) toSourceDocument(feedURL string) SourceDocument {+ body := e.renderText()+ fingerprint := rssFingerprint(body)+ metadata := map[string]any{"feed_url": feedURL}+ if e.link != "" {+ metadata["link"] = e.link+ }+ if e.author != "" {+ metadata["author"] = e.author+ }+ if len(e.categories) > 0 {+ metadata["categories"] = e.categories+ }+ return SourceDocument{+ SourceID: e.sourceID(),+ SemanticIdentifier: e.semanticIdentifier(),+ Extension: ".txt",+ Blob: []byte(body),+ UpdatedAt: e.updatedAt,+ SizeBytes: int64(len(body)),+ Metadata: metadata,+ Fingerprint: fingerprint,+ }+}++// renderText builds the plain-text RSS document body.+func (e rssEntry) renderText() string {+ parts := []string{e.semanticIdentifier()}+ body := htmlToText(firstNonEmpty(e.content, e.summary))+ if body != "" {+ parts = append(parts, body)+ }+ return strings.Join(parts, "\n\n")+}++type rssRoot struct {+ XMLName xml.Name+ Channel rssChannel `xml:"channel"`+}++type rssChannel struct {+ Items []rssXMLItem `xml:"item"`+}++type rssXMLItem struct {+ GUID string `xml:"guid"`+ Title string `xml:"title"`+ Link string `xml:"link"`+ Author string `xml:"author"`+ Creator string `xml:"creator"`+ PubDate string `xml:"pubDate"`+ Published string `xml:"published"`+ Updated string `xml:"updated"`+ Description string `xml:"description"`+ Summary string `xml:"summary"`+ Encoded string `xml:"encoded"`+ Categories []rssCategory `xml:"category"`+}++type rssCategory struct {+ Term string `xml:"term,attr"`+ Label string `xml:"label,attr"`+ Text string `xml:",chardata"`+}++type atomFeed struct {+ Entries []atomEntry `xml:"entry"`+}++type atomEntry struct {+ ID string `xml:"id"`+ Title string `xml:"title"`+ Updated string `xml:"updated"`+ Published string `xml:"published"`+ Summary atomText `xml:"summary"`+ Content atomText `xml:"content"`+ Links []atomLink `xml:"link"`+ Author atomPerson `xml:"author"`+ Categories []atomCategory `xml:"category"`+}++type atomText struct {+ Text string `xml:",innerxml"`+}++type atomLink struct {+ Href string `xml:"href,attr"`+ Rel string `xml:"rel,attr"`+}++type atomPerson struct {+ Name string `xml:"name"`+}++type atomCategory struct {+ Term string `xml:"term,attr"`+ Label string `xml:"label,attr"`+}++// parseRSSFeed parses RSS 2.0 and Atom payloads.+func parseRSSFeed(data []byte) ([]rssEntry, error) {+ decoder := xml.NewDecoder(bytes.NewReader(data))+ for {+ token, err := decoder.Token()+ if err != nil {+ return nil, fmt.Errorf("failed to parse RSS feed: %w", err)+ }+ if start, ok := token.(xml.StartElement); ok {+ switch strings.ToLower(start.Name.Local) {+ case "rss", "rdf":+ return parseRSSXML(data)+ case "feed":+ return parseAtomXML(data)+ default:+ return nil, fmt.Errorf("failed to parse RSS feed")+ }+ }+ }+}++// parseRSSXML parses an RSS-style feed document.+func parseRSSXML(data []byte) ([]rssEntry, error) {+ var root rssRoot+ if err := xml.Unmarshal(data, &root); err != nil {+ return nil, fmt.Errorf("failed to parse RSS feed: %w", err)+ }+ entries := make([]rssEntry, 0, len(root.Channel.Items))+ for _, item := range root.Channel.Items {+ entry := rssEntry{+ id: strings.TrimSpace(item.GUID),+ link: strings.TrimSpace(item.Link),+ title: strings.TrimSpace(item.Title),+ author: firstNonEmpty(item.Author, item.Creator),+ categories: rssCategories(item.Categories),+ content: firstNonEmpty(item.Encoded, item.Description),+ summary: firstNonEmpty(item.Summary, item.Description),+ updatedAt: parseFeedTime(firstNonEmpty(item.Updated, item.Published, item.PubDate)),+ }+ entries = append(entries, entry.withDefaults())+ }+ return entries, nil+}++// parseAtomXML parses an Atom feed document.+func parseAtomXML(data []byte) ([]rssEntry, error) {+ var feed atomFeed+ if err := xml.Unmarshal(data, &feed); err != nil {+ return nil, fmt.Errorf("failed to parse RSS feed: %w", err)+ }+ entries := make([]rssEntry, 0, len(feed.Entries))+ for _, item := range feed.Entries {+ entry := rssEntry{+ id: strings.TrimSpace(item.ID),+ link: atomEntryLink(item.Links),+ title: htmlToText(item.Title),+ author: strings.TrimSpace(item.Author.Name),+ categories: atomCategories(item.Categories),+ content: item.Content.Text,+ summary: item.Summary.Text,+ updatedAt: parseFeedTime(firstNonEmpty(item.Updated, item.Published)),+ }+ entries = append(entries, entry.withDefaults())+ }+ return entries, nil+}++// withDefaults fills missing stable fields with deterministic fallbacks.+func (e rssEntry) withDefaults() rssEntry {+ return e+}++// rssFingerprint returns a deterministic body fingerprint.+func rssFingerprint(body string) string {+ sum := xxh3.Hash128([]byte(body)).Bytes()+ return hex.EncodeToString(sum[:])+}++// rssCategories returns RSS category labels.+func rssCategories(categories []rssCategory) []string {+ out := make([]string, 0, len(categories))+ for _, category := range categories {+ value := firstNonEmpty(category.Term, category.Label, category.Text)+ if value != "" {+ out = append(out, value)+ }+ }+ return out+}++// atomCategories returns Atom category labels.+func atomCategories(categories []atomCategory) []string {+ out := make([]string, 0, len(categories))+ for _, category := range categories {+ value := firstNonEmpty(category.Label, category.Term)+ if value != "" {+ out = append(out, value)+ }+ }+ return out+}++// atomEntryLink returns the preferred Atom link.+func atomEntryLink(links []atomLink) string {+ for _, link := range links {+ if link.Href != "" && (link.Rel == "" || link.Rel == "alternate") {+ return strings.TrimSpace(link.Href)+ }+ }+ if len(links) > 0 {+ return strings.TrimSpace(links[0].Href)+ }+ return ""+}++// htmlToText removes markup and normalizes text spacing.+func htmlToText(value string) string {+ value = rssHTMLScriptStyleRE.ReplaceAllString(value, "")+ value = rssHTMLTagRE.ReplaceAllString(value, "\n")+ value = html.UnescapeString(value)+ value = strings.ReplaceAll(value, "\r\n", "\n")+ value = strings.ReplaceAll(value, "\r", "\n")+ lines := strings.Split(value, "\n")+ for i, line := range lines {+ lines[i] = strings.TrimSpace(rssWhitespaceRE.ReplaceAllString(line, " "))+ }+ value = strings.Join(lines, "\n")+ value = rssNewlineRE.ReplaceAllString(value, "\n\n")+ return strings.TrimSpace(value)+}++// parseFeedTime parses common RSS and Atom time formats.+func parseFeedTime(value string) time.Time {+ value = strings.TrimSpace(value)+ if value == "" {+ return time.Time{}+ }+ layouts := []string{+ time.RFC1123Z,+ time.RFC1123,+ time.RFC3339Nano,+ time.RFC3339,+ "Mon, 02 Jan 2006 15:04:05 -0700",+ "Mon, 2 Jan 2006 15:04:05 -0700",+ "2006-01-02 15:04:05",+ }+ for _, layout := range layouts {+ if parsed, err := time.Parse(layout, value); err == nil {+ return parsed.UTC()+ }+ }+ return time.Time{}+}++// firstNonEmpty returns the first non-empty trimmed string.+func firstNonEmpty(values ...string) string {+ for _, value := range values {+ if strings.TrimSpace(value) != "" {+ return strings.TrimSpace(value)+ }+ }+ return ""+}++// configInt reads a positive integer-like config value.+func configInt(value any, fallback int) int {+ switch typed := value.(type) {+ case int:+ if typed > 0 {+ return typed+ }+ case int64:+ if typed > 0 {+ return int(typed)+ }+ case float64:+ if typed > 0 {+ return int(typed)+ }+ case string:+ if parsed, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil && parsed > 0 {+ return parsed+ }+ }+ return fallback+}diff --git a/internal/syncer/connector/rss_test.go b/internal/syncer/connector/rss_test.gonew file mode 100644index 0000000000..bfae3c2adc--- /dev/null+++ b/internal/syncer/connector/rss_test.go@@ -0,0 +1,141 @@+package connector++import (+ "context"+ "crypto/md5"+ "encoding/hex"+ "errors"+ "io"+ "testing"+ "time"+)++// TestRSSConnectorOpenSyncFullAndIncremental verifies full and windowed sync.+func TestRSSConnectorOpenSyncFullAndIncremental(t *testing.T) {+ connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 1})+ if err != nil {+ t.Fatalf("NewRSSConnector failed: %v", err)+ }+ connector.fetchFeed = staticRSSFeed++ fullSession, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-05T00:00:00Z")})+ if err != nil {+ t.Fatalf("OpenSync full failed: %v", err)+ }+ first, err := fullSession.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != expectedRSSSourceID("entry-old") {+ t.Fatalf("unexpected first full batch: %+v", first.Documents)+ }+ second, err := fullSession.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch second failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != expectedRSSSourceID("entry-new") {+ t.Fatalf("unexpected second full batch: %+v", second.Documents)+ }+ if _, err = fullSession.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }++ start := mustTime(t, "2026-01-02T00:00:00Z")+ incrementalSession, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: mustTime(t, "2026-01-04T00:00:00Z")})+ if err != nil {+ t.Fatalf("OpenSync incremental failed: %v", err)+ }+ batch, err := incrementalSession.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch incremental failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("incremental batch len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != expectedRSSSourceID("entry-new") {+ t.Fatalf("incremental source id = %s", doc.SourceID)+ }+ if doc.SemanticIdentifier != "New title" {+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)+ }+ if string(doc.Blob) != "New title\n\nNew body" {+ t.Fatalf("blob = %q", string(doc.Blob))+ }+ if doc.Metadata["link"] != "https://example.com/new" {+ t.Fatalf("link metadata = %v", doc.Metadata["link"])+ }+}++// TestRSSConnectorOpenPrune verifies complete slim snapshot generation.+func TestRSSConnectorOpenPrune(t *testing.T) {+ connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 10})+ if err != nil {+ t.Fatalf("NewRSSConnector failed: %v", err)+ }+ connector.fetchFeed = staticRSSFeed+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 3 {+ t.Fatalf("prune snapshot len = %d, want 3", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != expectedRSSSourceID("entry-old") ||+ batch.Documents[1].SourceID != expectedRSSSourceID("entry-new") ||+ batch.Documents[2].SourceID != expectedRSSSourceID("entry-future") {+ t.Fatalf("unexpected prune ids: %+v", batch.Documents)+ }+}++// staticRSSFeed returns a fixed RSS feed for tests.+func staticRSSFeed(ctx context.Context, feedURL string) ([]byte, error) {+ return []byte(`<?xml version="1.0" encoding="UTF-8"?>+<rss version="2.0">+ <channel>+ <title>Feed</title>+ <item>+ <guid>entry-old</guid>+ <title>Old title</title>+ <link>https://example.com/old</link>+ <pubDate>Thu, 01 Jan 2026 00:00:00 +0000</pubDate>+ <description><![CDATA[<p>Old body</p>]]></description>+ </item>+ <item>+ <guid>entry-new</guid>+ <title>New title</title>+ <link>https://example.com/new</link>+ <pubDate>Sat, 03 Jan 2026 00:00:00 +0000</pubDate>+ <description><![CDATA[<p>New body</p>]]></description>+ <category>news</category>+ </item>+ <item>+ <guid>entry-future</guid>+ <title>Future title</title>+ <link>https://example.com/future</link>+ <pubDate>Tue, 06 Jan 2026 00:00:00 +0000</pubDate>+ <description><![CDATA[<p>Future body</p>]]></description>+ </item>+ </channel>+</rss>`), nil+}++// expectedRSSSourceID returns Python-compatible RSS source ID.+func expectedRSSSourceID(value string) string {+ sum := md5.Sum([]byte(value))+ return "rss:" + hex.EncodeToString(sum[:])+}++// mustTime parses a test timestamp.+func mustTime(t *testing.T, value string) time.Time {+ t.Helper()+ parsed, err := time.Parse(time.RFC3339, value)+ if err != nil {+ t.Fatalf("parse time %q: %v", value, err)+ }+ return parsed+}diff --git a/internal/syncer/connector_lock.go b/internal/syncer/connector_lock.gonew file mode 100644index 0000000000..4bacaffa78--- /dev/null+++ b/internal/syncer/connector_lock.go@@ -0,0 +1,54 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import "sync"++// ConnectorLocker serializes work for a connector.+type ConnectorLocker interface {+ TryLock(connectorID string) bool+ Unlock(connectorID string)+}++// ConnectorLock is a process-local connector mutex registry.+type ConnectorLock struct {+ mu sync.Mutex+ locked map[string]struct{}+}++// NewConnectorLock creates an empty process-local connector lock.+func NewConnectorLock() *ConnectorLock {+ return &ConnectorLock{locked: map[string]struct{}{}}+}++// TryLock attempts to acquire the connector lock without blocking.+func (l *ConnectorLock) TryLock(connectorID string) bool {+ l.mu.Lock()+ defer l.mu.Unlock()+ if _, ok := l.locked[connectorID]; ok {+ return false+ }+ l.locked[connectorID] = struct{}{}+ return true+}++// Unlock releases the connector lock.+func (l *ConnectorLock) Unlock(connectorID string) {+ l.mu.Lock()+ defer l.mu.Unlock()+ delete(l.locked, connectorID)+}diff --git a/internal/syncer/prune_runner.go b/internal/syncer/prune_runner.gonew file mode 100644index 0000000000..c8435fc3ba--- /dev/null+++ b/internal/syncer/prune_runner.go@@ -0,0 +1,80 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "errors"+ "io"+ "ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector"+)++// PruneRunner executes one PRUNE task after collecting a full slim snapshot.+type PruneRunner struct {+ taskService *service.SyncTaskService+ pruneService *service.SyncPruneService+}++// NewPruneRunner creates a PRUNE runner.+func NewPruneRunner(taskService *service.SyncTaskService, pruneService *service.SyncPruneService) *PruneRunner {+ return &PruneRunner{taskService: taskService, pruneService: pruneService}+}++// Run collects the full prune snapshot before deleting stale documents.+func (r *PruneRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {+ if r.pruneService == nil {+ return errors.New("prune service is not configured")+ }+ session, err := connector.OpenPrune(ctx, syncerconnector.PruneRequest{+ TaskID: taskContext.Task.ID,+ ConnectorID: taskContext.Connector.ID,+ KBID: taskContext.Knowledgebase.ID,+ })+ if err != nil {+ return err+ }+ defer session.Close()++ if err = r.pruneService.ClearSnapshot(ctx, taskContext.Task.ID); err != nil {+ return err+ }+ clearSnapshot := func() {+ _ = r.pruneService.ClearSnapshot(context.WithoutCancel(ctx), taskContext.Task.ID)+ }+ for {+ batch, nextErr := session.NextBatch(ctx)+ if errors.Is(nextErr, io.EOF) {+ break+ }+ if nextErr != nil {+ clearSnapshot()+ return nextErr+ }+ if err = r.pruneService.AddSnapshotBatch(ctx, taskContext.Task.ID, batch.Documents); err != nil {+ clearSnapshot()+ return err+ }+ }++ removed, err := r.pruneService.DeleteStale(ctx, taskContext)+ if err != nil {+ clearSnapshot()+ return err+ }+ return r.taskService.CompletePrune(ctx, taskContext, removed)+}diff --git a/internal/syncer/recovery.go b/internal/syncer/recovery.gonew file mode 100644index 0000000000..1fea849a1f--- /dev/null+++ b/internal/syncer/recovery.go@@ -0,0 +1,28 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "ragflow/internal/service"+ "time"+)++// RecoverStaleRunning restores timed-out running sync tasks to schedule.+func RecoverStaleRunning(ctx context.Context, taskService *service.SyncTaskService, now time.Time) error {+ return taskService.RecoverStaleRunning(ctx, now)+}diff --git a/internal/syncer/scheduler.go b/internal/syncer/scheduler.gonew file mode 100644index 0000000000..0cc775b646--- /dev/null+++ b/internal/syncer/scheduler.go@@ -0,0 +1,90 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "errors"+ "ragflow/internal/common"+ "ragflow/internal/service"+ "time"+)++// TaskEnvelope is the only payload sent through the task queue.+type TaskEnvelope struct {+ TaskID string+}++// Scheduler scans due work and enqueues claimed task IDs.+type Scheduler struct {+ pollInterval time.Duration+ queue chan<- TaskEnvelope+ taskService *service.SyncTaskService+}++// NewScheduler creates a global scheduler for datasource sync tasks.+func NewScheduler(pollInterval time.Duration, queue chan<- TaskEnvelope, taskService *service.SyncTaskService) *Scheduler {+ return &Scheduler{pollInterval: pollInterval, queue: queue, taskService: taskService}+}++// Run starts recovery and periodic task discovery.+func (s *Scheduler) Run(ctx context.Context) error {+ if err := s.scan(ctx); err != nil && ctx.Err() == nil {+ common.Error("syncer scheduler scan failed", err)+ }+ ticker := time.NewTicker(s.pollInterval)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-ticker.C:+ if err := s.scan(ctx); err != nil && ctx.Err() == nil {+ common.Error("syncer scheduler scan failed", err)+ }+ }+ }+}++// scan claims due tasks and places their IDs on the bounded queue.+func (s *Scheduler) scan(ctx context.Context) error {+ now := time.Now()+ if err := s.taskService.RecoverStaleRunning(ctx, now); err != nil {+ return err+ }+ tasks, err := s.taskService.ListDueTasks(ctx, now)+ if err != nil {+ return err+ }+ var claimErr error+ for _, task := range tasks {+ claimed, err := s.taskService.Claim(ctx, task.ID)+ if err != nil {+ claimErr = errors.Join(claimErr, err)+ continue+ }+ if !claimed {+ continue+ }+ select {+ case <-ctx.Done():+ return ctx.Err()+ case s.queue <- TaskEnvelope{TaskID: task.ID}:+ }+ }+ return claimErr+}diff --git a/internal/syncer/sync_runner.go b/internal/syncer/sync_runner.gonew file mode 100644index 0000000000..41edd8a0a3--- /dev/null+++ b/internal/syncer/sync_runner.go@@ -0,0 +1,214 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "errors"+ "fmt"+ "io"+ "math/rand"+ "ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector"+ "sync"+ "time"+)++// SyncRunner executes one SYNC task with serial batches and parallel items.+type SyncRunner struct {+ config TaskCoordinatorConfig+ taskService *service.SyncTaskService+ sink service.DocumentSink+ idResolver *service.DocumentIDResolver+ globalItems chan struct{}+}++// NewSyncRunner creates a SYNC runner.+func NewSyncRunner(config TaskCoordinatorConfig, taskService *service.SyncTaskService, sink service.DocumentSink, idResolver *service.DocumentIDResolver, globalItems chan struct{}) *SyncRunner {+ return &SyncRunner{config: config, taskService: taskService, sink: sink, idResolver: idResolver, globalItems: globalItems}+}++// Run executes all sync batches and commits the final waterline.+func (r *SyncRunner) Run(ctx context.Context, taskContext service.SyncTaskContext, connector syncerconnector.Connector) error {+ if r.sink == nil {+ return errors.New("document sink is not configured")+ }+ windowEnd := time.Now().UTC()++ var windowStart *time.Time+ if !service.IsFromBeginning(taskContext.Task.FromBeginning) {+ windowStart = taskContext.Task.PollRangeStart+ }++ session, err := connector.OpenSync(ctx, syncerconnector.SyncRequest{+ TaskID: taskContext.Task.ID,+ ConnectorID: taskContext.Connector.ID,+ KBID: taskContext.Knowledgebase.ID,+ FromBeginning: windowStart == nil,+ WindowStart: windowStart,+ WindowEnd: windowEnd,+ })+ if err != nil {+ return err+ }+ defer session.Close()++ sourceType := service.SourceType(taskContext.Connector.Source, taskContext.Connector.ID)+ candidateEnd := windowStart+ stats := service.SyncStats{}+ for {+ batch, nextErr := session.NextBatch(ctx)+ if errors.Is(nextErr, io.EOF) {+ break+ }+ if nextErr != nil {+ return nextErr+ }+ for _, doc := range batch.Documents {+ if candidateEnd == nil || doc.UpdatedAt.After(*candidateEnd) {+ updatedAt := doc.UpdatedAt+ candidateEnd = &updatedAt+ }+ }+ batchStats, err := r.processBatch(ctx, taskContext, sourceType, session, batch)+ if err != nil {+ return err+ }+ stats.Add(batchStats)+ }++ if candidateEnd == nil {+ candidateEnd = &windowEnd+ }+ return r.taskService.CompleteSync(ctx, taskContext, *candidateEnd, stats)+}++// processBatch processes one batch with bounded document concurrency.+func (r *SyncRunner) processBatch(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, batch syncerconnector.SyncBatch) (service.SyncStats, error) {+ sem := make(chan struct{}, r.config.PerTaskItemConcurrency)+ results := make(chan service.DocumentUpsertResult, len(batch.Documents))+ errs := make(chan error, len(batch.Documents))++ var wg sync.WaitGroup+ for _, sourceDocument := range batch.Documents {+ sourceDocument := sourceDocument++ sem <- struct{}{}+ wg.Add(1)+ go func() {+ defer wg.Done()+ defer func() { <-sem }()+ result, err := r.processDocumentWithRetry(ctx, taskContext, sourceType, session, sourceDocument)+ if err != nil {+ errs <- err+ return+ }+ results <- result+ }()+ }++ wg.Wait()+ close(results)+ close(errs)++ if len(errs) > 0 {+ return service.SyncStats{}, <-errs+ }+ stats := service.SyncStats{}++ for result := range results {+ stats.AddResult(result)+ }+ return stats, nil+}++// processDocumentWithRetry retries transient item failures.+func (r *SyncRunner) processDocumentWithRetry(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, sourceDocument syncerconnector.SourceDocument) (service.DocumentUpsertResult, error) {+ var lastErr error+ for attempt := 1; attempt <= r.config.ItemRetryCount; attempt++ {+ result, err := r.processDocument(ctx, taskContext, sourceType, session, sourceDocument)+ if err == nil {+ return result, nil+ }++ lastErr = err+ if !service.IsRetryable(err) || attempt == r.config.ItemRetryCount {+ break+ }++ shift := attempt - 1+ if shift > 30 {+ shift = 30+ }+ delay := r.config.ItemRetryBaseDelay * time.Duration(1<<shift)+ if delay < 0 {+ delay = time.Hour+ }+ if jitterMax := r.config.ItemRetryBaseDelay / 2; jitterMax > 0 {+ delay += time.Duration(rand.Int63n(int64(jitterMax)))+ }++ select {+ case <-ctx.Done():+ return service.DocumentUpsertResult{}, ctx.Err()+ case <-time.After(delay):+ }+ }+ return service.DocumentUpsertResult{}, lastErr+}++// processDocument resolves IDs, skips unchanged fingerprints, fetches blobs, and upserts.+func (r *SyncRunner) processDocument(ctx context.Context, taskContext service.SyncTaskContext, sourceType string, session syncerconnector.SyncSession, sourceDocument syncerconnector.SourceDocument) (service.DocumentUpsertResult, error) {+ if r.globalItems != nil {+ select {+ case <-ctx.Done():+ return service.DocumentUpsertResult{}, ctx.Err()+ case r.globalItems <- struct{}{}:+ defer func() { <-r.globalItems }()+ }+ }++ resolved, err := r.idResolver.Resolve(ctx, taskContext.Knowledgebase.ID, taskContext.Connector.ID, sourceType, sourceDocument.SourceID)+ if err != nil {+ return service.DocumentUpsertResult{}, err+ }+ if sourceDocument.Fingerprint != "" && resolved.StoredFingerprint == sourceDocument.Fingerprint {+ return service.DocumentUpsertResult{DocID: resolved.DocID, Action: service.DocumentActionSkipped}, nil+ }++ if len(sourceDocument.Blob) == 0 && sourceDocument.FetchRef != nil {+ fetcher, ok := session.(syncerconnector.Fetcher)+ if !ok {+ return service.DocumentUpsertResult{}, fmt.Errorf("connector session cannot fetch %s", sourceDocument.FetchRef.Key)+ }+ blob, err := fetcher.Fetch(ctx, *sourceDocument.FetchRef)+ if err != nil {+ return service.DocumentUpsertResult{}, err+ }+ sourceDocument.Blob = blob+ }++ return r.sink.Upsert(ctx, service.DocumentUpsertInput{+ TaskContext: taskContext,+ SourceType: sourceType,+ DocumentID: resolved.DocID,+ LegacyID: resolved.LegacyID,+ NewID: resolved.NewID,+ SourceDocument: sourceDocument,+ AutoParse: taskContext.Connector2Kb.AutoParse != "0",+ })+}diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.goindex 6df9e539b9..23cf67f88d 100644--- a/internal/syncer/syncer.go+++ b/internal/syncer/syncer.go@@ -18,127 +18,136 @@ package syncerimport ("context"- "fmt"- "ragflow/internal/common"- "ragflow/internal/dao"- "ragflow/internal/entity"- "ragflow/internal/utility"+ "errors""sync""time"- "go.uber.org/zap"+ "ragflow/internal/dao"+ "ragflow/internal/service"+ documentservice "ragflow/internal/service/document"+ syncerconnector "ragflow/internal/syncer/connector"+ "ragflow/internal/utility")-// Syncer periodically polls the sync_logs table and dispatches due-// sync/prune tasks to a fixed-size worker pool.+// Syncer owns the scheduler, task queue, and bounded worker pool.type Syncer struct {- id string- maxConcurrency int- pollInterval time.Duration // how often each worker queries for due tasks+ id string+ config Config+ queue chan TaskEnvelope+ scheduler *Scheduler+ worker *TaskWorker+ cancel context.CancelFunc+ workerGroup sync.WaitGroup+ stopOnce sync.Once+ ShutdownCh chan struct{}+}- ctx context.Context- cancel context.CancelFunc+// NewSyncer creates a server-compatible syncer with default dependencies.+func NewSyncer(maxConcurrency int, pollInterval time.Duration) *Syncer {+ // init the config+ config := DefaultConfig()+ config.TaskConcurrency = maxConcurrency+ config.PollInterval = pollInterval- workerWg sync.WaitGroup+ taskDAO := dao.NewSyncTaskDAO(nil)+ registry := syncerconnector.NewRegistry()- // ShutdownCh is closed when Stop() completes.- ShutdownCh chan struct{}-}+ registerBuiltInConnectors(registry)-// NewSyncer creates a syncer with the given concurrency and poll interval.-func NewSyncer(maxConcurrency int, pollInterval time.Duration) *Syncer {- ctx, cancel := context.WithCancel(context.Background())- return &Syncer{- id: utility.GenerateUUID(),- maxConcurrency: maxConcurrency,- pollInterval: pollInterval,- ctx: ctx,- cancel: cancel,- ShutdownCh: make(chan struct{}),- }+ documentService := documentservice.NewDocumentService()+ pruneService := service.NewSyncPruneService(dao.NewSyncPruneSnapshotDAO(taskDAO.DB()), documentService, nil)++ return New(config, taskDAO, registry, documentService, pruneService)}-// Start launches maxConcurrency worker goroutines.-func (s *Syncer) Start() error {- common.Info(fmt.Sprintf("Syncer %s starting with %d workers (poll every %v)",- s.id, s.maxConcurrency, s.pollInterval))+// New creates a datasource syncer from explicit dependencies.+func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService) *Syncer {+ config = config.Normalize()+ queue := make(chan TaskEnvelope, config.TaskQueueSize)+ locker := NewConnectorLock()- for i := 0; i < s.maxConcurrency; i++ {- s.workerWg.Add(1)- go s.workerLoop(i)- }- return nil-}+ globalItems := make(chan struct{}, config.GlobalItemConcurrency)+ taskService := service.NewSyncTaskService(taskDAO)+ idResolver := service.NewDocumentIDResolver(service.NewGormDocumentStore())-// Stop cancels all workers and waits for them to finish.-func (s *Syncer) Stop() {- common.Info(fmt.Sprintf("Stopping syncer %s", s.id))- s.cancel()- s.workerWg.Wait()- close(s.ShutdownCh)- common.Info(fmt.Sprintf("Syncer %s stopped", s.id))+ coordinator := NewTaskCoordinator(TaskCoordinatorConfig{+ PerTaskItemConcurrency: config.PerTaskItemConcurrency,+ ItemRetryCount: config.ItemRetryCount,+ ItemRetryBaseDelay: config.ItemRetryBaseDelay,+ }, taskService, registry, sink, pruneService, idResolver, globalItems)++ return &Syncer{+ id: utility.GenerateUUID(),+ config: config,+ queue: queue,+ scheduler: NewScheduler(config.PollInterval, queue, taskService),+ worker: NewTaskWorker(queue, taskService, coordinator, locker),+ ShutdownCh: make(chan struct{}),+ }}+// ID returns this syncer process ID.func (s *Syncer) ID() string {+ if s == nil {+ return ""+ }return s.id}-// workerLoop periodically polls the DB for due tasks until ctx is cancelled.-func (s *Syncer) workerLoop(workerID int) {- defer s.workerWg.Done()- common.Debug(fmt.Sprintf("Syncer worker %d started", workerID))-- ticker := time.NewTicker(s.pollInterval)- defer ticker.Stop()-- for {- select {- case <-s.ctx.Done():- common.Debug(fmt.Sprintf("Syncer worker %d exiting (ctx cancelled)", workerID))- return- case <-ticker.C:- s.pollAndExecute(workerID)- }- }+// Start launches the scheduler and task workers with a background context.+func (s *Syncer) Start() error {+ return s.StartContext(context.Background())}-// pollAndExecute queries due sync & prune tasks, picks one, and runs it.-func (s *Syncer) pollAndExecute(workerID int) {- common.Info(fmt.Sprintf("Syncer worker %d polling for due tasks", workerID))-}+// StartContext launches the scheduler and task workers.+func (s *Syncer) StartContext(ctx context.Context) error {+ if s == nil {+ return errors.New("syncer is nil")+ }-// executeSyncTask runs a sync task.-func (s *Syncer) executeSyncTask(task *entity.SyncLogs) {- common.Info("Executing sync task",- zap.String("task_id", task.ID),- zap.String("connector_id", task.ConnectorID),- zap.String("kb_id", task.KbID))- // TODO: implement actual data-source-specific sync logic.- // For now, mark done.- s.markTaskDone(task.ID, task.ConnectorID)+ runCtx, cancel := context.WithCancel(ctx)+ s.cancel = cancel+ s.workerGroup.Add(2)++ go func() {+ defer s.workerGroup.Done()+ _ = s.scheduler.Run(runCtx)+ }()+ go func() {+ defer s.workerGroup.Done()+ s.worker.Run(runCtx, s.config.TaskConcurrency)+ }()+ return nil}-// executePruneTask runs a prune (delete stale docs) task.-func (s *Syncer) executePruneTask(task *entity.SyncLogs) {- common.Info("Executing prune task",- zap.String("task_id", task.ID),- zap.String("connector_id", task.ConnectorID),- zap.String("kb_id", task.KbID))- // TODO: implement actual prune logic.- s.markTaskDone(task.ID, task.ConnectorID)+// Stop cancels the scheduler and waits for workers to exit.+func (s *Syncer) Stop() {+ if s == nil {+ return+ }+ s.stopOnce.Do(func() {+ if s.cancel != nil {+ s.cancel()+ }+ s.workerGroup.Wait()+ close(s.ShutdownCh)+ })}-// markTaskDone updates task and connector status to DONE.-func (s *Syncer) markTaskDone(taskID, connectorID string) {- db := dao.GetDB()- now := time.Now().Local()-- db.Model(&entity.SyncLogs{}).Where("id = ?", taskID).Updates(map[string]interface{}{- "status": string(entity.TaskStatusDone),- "update_time": now,+// registerBuiltInConnectors registers datasource connectors available in the server binary.+func registerBuiltInConnectors(registry *syncerconnector.Registry) {+ registry.Register("rss", func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {+ row, ok := taskContext.(dao.SyncTaskContext)+ if !ok {+ return nil, errors.New("rss connector received an invalid task context")+ }+ return syncerconnector.NewRSSConnector(map[string]any(row.Connector.Config))})- db.Model(&entity.Connector{}).Where("id = ?", connectorID).Updates(map[string]interface{}{- "status": string(entity.TaskStatusDone),- "update_time": now,+ registry.Register("github", func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {+ row, ok := taskContext.(dao.SyncTaskContext)+ if !ok {+ return nil, errors.New("github connector received an invalid task context")+ }+ return syncerconnector.NewGitHubConnector(map[string]any(row.Connector.Config))})}diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.gonew file mode 100644index 0000000000..9c24e43d31--- /dev/null+++ b/internal/syncer/syncer_test.go@@ -0,0 +1,502 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "errors"+ "io"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector"+ connectormock "ragflow/internal/syncer/connector/mock"+ "sync"+ "testing"+ "time"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"+ "gorm.io/gorm/logger"+)++// fakeSink records document writes for coordinator tests.+type fakeSink struct {+ mu sync.Mutex+ delay time.Duration+ current int+ maxConcurrent int+ calls []service.DocumentUpsertInput+ errBySourceID map[string]error+ autoParseByDoc map[string]bool+}++// Upsert records one document write.+func (s *fakeSink) Upsert(ctx context.Context, input service.DocumentUpsertInput) (service.DocumentUpsertResult, error) {+ s.mu.Lock()+ s.current+++ if s.current > s.maxConcurrent {+ s.maxConcurrent = s.current+ }+ s.calls = append(s.calls, input)+ if s.autoParseByDoc == nil {+ s.autoParseByDoc = map[string]bool{}+ }+ s.autoParseByDoc[input.SourceDocument.SourceID] = input.AutoParse+ s.mu.Unlock()+ if s.delay > 0 {+ select {+ case <-ctx.Done():+ return service.DocumentUpsertResult{}, ctx.Err()+ case <-time.After(s.delay):+ }+ }+ s.mu.Lock()+ s.current--+ s.mu.Unlock()+ if err := s.errBySourceID[input.SourceDocument.SourceID]; err != nil {+ return service.DocumentUpsertResult{}, err+ }+ return service.DocumentUpsertResult{DocID: input.DocumentID, Action: service.DocumentActionAdded}, nil+}++// callCount returns the number of sink calls.+func (s *fakeSink) callCount() int {+ s.mu.Lock()+ defer s.mu.Unlock()+ return len(s.calls)+}++// fakeStore provides document IDs and fingerprints.+type fakeStore struct {+ ids map[string]struct{}+ fingerprints map[string]string+}++// ListIDs returns configured IDs.+func (s fakeStore) ListIDs(ctx context.Context, kbID, sourceType string) (map[string]struct{}, error) {+ if s.ids == nil {+ return map[string]struct{}{}, nil+ }+ return s.ids, nil+}++// GetFingerprintsByIDs returns configured fingerprints for requested IDs.+func (s fakeStore) GetFingerprintsByIDs(ctx context.Context, kbID, sourceType string, ids []string) (map[string]string, error) {+ result := make(map[string]string, len(ids))+ if s.fingerprints == nil {+ return result, nil+ }+ for _, id := range ids {+ if fingerprint, ok := s.fingerprints[id]; ok {+ result[id] = fingerprint+ }+ }+ return result, nil+}++// fakeDeleter records document deletes.+type fakeDeleter struct {+ mu sync.Mutex+ deleted []string+}++// DeleteDocument records one delete.+func (d *fakeDeleter) DeleteDocument(ctx context.Context, docID string) error {+ d.mu.Lock()+ defer d.mu.Unlock()+ d.deleted = append(d.deleted, docID)+ return nil+}++// setupSyncerDB creates a SQLite test database.+func setupSyncerDB(t *testing.T) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err = db.AutoMigrate(&entity.Connector{}, &entity.Connector2Kb{}, &entity.Knowledgebase{}, &entity.SyncLogs{}, &entity.Document{}, &dao.SyncPruneSnapshot{}); err != nil {+ t.Fatalf("migrate sqlite: %v", err)+ }+ orig := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = orig })+ return db+}++// insertTaskContext inserts one connector, mapping, KB, and sync task.+func insertTaskContext(t *testing.T, db *gorm.DB, connectorID, kbID, taskID, taskType string) {+ t.Helper()+ now := time.Now().Add(-time.Hour).Truncate(time.Second)+ ts := now.UnixMilli()+ fromBeginning := "0"+ var existingConnector int64+ if err := db.Model(&entity.Connector{}).Where("id = ?", connectorID).Count(&existingConnector).Error; err != nil {+ t.Fatalf("count connector: %v", err)+ }+ if existingConnector == 0 {+ if err := db.Create(&entity.Connector{+ ID: connectorID,+ TenantID: "tenant-1",+ Name: connectorID,+ Source: "mock",+ InputType: "poll",+ Config: entity.JSONMap{"sync_deleted_files": true},+ RefreshFreq: 0,+ PruneFreq: 0,+ TimeoutSecs: 1,+ Status: dao.SyncStatusSchedule,+ BaseModel: entity.BaseModel{UpdateDate: &now, UpdateTime: &ts},+ }).Error; err != nil {+ t.Fatalf("insert connector: %v", err)+ }+ }+ if err := db.Create(&entity.Knowledgebase{+ ID: kbID,+ TenantID: "tenant-1",+ Name: kbID,+ EmbdID: "embd",+ CreatedBy: "tenant-1",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ }).Error; err != nil {+ t.Fatalf("insert kb: %v", err)+ }+ if err := db.Create(&entity.Connector2Kb{ID: connectorID + kbID, ConnectorID: connectorID, KbID: kbID, AutoParse: "1"}).Error; err != nil {+ t.Fatalf("insert mapping: %v", err)+ }+ if err := db.Create(&entity.SyncLogs{+ ID: taskID,+ ConnectorID: connectorID,+ KbID: kbID,+ TaskType: taskType,+ Status: dao.SyncStatusSchedule,+ FromBeginning: &fromBeginning,+ TimeStarted: &now,+ ErrorMsg: "",+ BaseModel: entity.BaseModel{UpdateDate: &now, UpdateTime: &ts},+ }).Error; err != nil {+ t.Fatalf("insert task: %v", err)+ }+}++// newTestRegistry creates a mock connector registry.+func newTestRegistry(connectors map[string]*connectormock.Connector) *syncerconnector.Registry {+ registry := syncerconnector.NewRegistry()+ registry.Register("mock", func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {+ row := taskContext.(dao.SyncTaskContext)+ return connectors[row.Connector.ID], nil+ })+ return registry+}++// newCoordinator creates a test coordinator.+func newCoordinator(taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore, perTask int) *TaskCoordinator {+ return NewTaskCoordinator(TaskCoordinatorConfig{PerTaskItemConcurrency: perTask, ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), make(chan struct{}, 16))+}++// TestSchedulerClaimsDueTasks verifies conditional task claiming.+func TestSchedulerClaimsDueTasks(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ queue := make(chan TaskEnvelope, 1)+ scheduler := NewScheduler(time.Hour, queue, taskService)+ if err := scheduler.scan(t.Context()); err != nil {+ t.Fatalf("scan: %v", err)+ }+ envelope := <-queue+ if envelope.TaskID != "task-1" {+ t.Fatalf("TaskID = %s, want task-1", envelope.TaskID)+ }+ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusRunning {+ t.Fatalf("status = %s, want running", task.Status)+ }+}++// TestWorkersRunDifferentConnectorsInParallel verifies task-level parallelism.+func TestWorkersRunDifferentConnectorsInParallel(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ insertTaskContext(t, db, "conn-2", "kb-2", "task-2", dao.TaskTypeSync)+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ now := time.Now()+ for _, id := range []string{"task-1", "task-2"} {+ claimed, err := taskDAO.ClaimTask(t.Context(), id, now)+ if err != nil || !claimed {+ t.Fatalf("claim %s: %v %v", id, claimed, err)+ }+ }+ sink := &fakeSink{delay: 120 * time.Millisecond}+ connectors := map[string]*connectormock.Connector{+ "conn-1": {SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: now}}}}},+ "conn-2": {SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "b", UpdatedAt: now}}}}},+ }+ queue := make(chan TaskEnvelope, 2)+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(connectors), sink, nil, fakeStore{}, 1), NewConnectorLock())+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ go worker.Run(ctx, 2)+ queue <- TaskEnvelope{TaskID: "task-1"}+ queue <- TaskEnvelope{TaskID: "task-2"}+ time.Sleep(40 * time.Millisecond)+ sink.mu.Lock()+ maxConcurrent := sink.maxConcurrent+ sink.mu.Unlock()+ if maxConcurrent < 2 {+ t.Fatalf("max concurrent sink calls = %d, want >= 2", maxConcurrent)+ }+}++// TestConnectorLockSerializesSameConnector verifies connector-level mutual exclusion.+func TestConnectorLockSerializesSameConnector(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ insertTaskContext(t, db, "conn-1", "kb-2", "task-2", dao.TaskTypeSync)+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ for _, id := range []string{"task-1", "task-2"} {+ if _, err := taskDAO.ClaimTask(t.Context(), id, time.Now()); err != nil {+ t.Fatalf("claim %s: %v", id, err)+ }+ }+ sink := &fakeSink{delay: 100 * time.Millisecond}+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: time.Now()}}}}}+ queue := make(chan TaskEnvelope, 2)+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 1), NewConnectorLock())+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ go worker.Run(ctx, 2)+ queue <- TaskEnvelope{TaskID: "task-1"}+ queue <- TaskEnvelope{TaskID: "task-2"}+ time.Sleep(180 * time.Millisecond)+ var scheduled int64+ if err := db.Model(&entity.SyncLogs{}).Where("connector_id = ? AND status = ?", "conn-1", dao.SyncStatusSchedule).Count(&scheduled).Error; err != nil {+ t.Fatalf("count scheduled: %v", err)+ }+ if scheduled == 0 {+ t.Fatalf("expected one same-connector task to be rescheduled")+ }+ sink.mu.Lock()+ maxConcurrent := sink.maxConcurrent+ sink.mu.Unlock()+ if maxConcurrent > 1 {+ t.Fatalf("same connector ran concurrently: %d", maxConcurrent)+ }+}++// TestSyncRunnerReadsBatchesSerially verifies the next batch waits for current items.+func TestSyncRunnerReadsBatchesSerially(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ taskDAO := dao.NewSyncTaskDAO(db)+ taskService := service.NewSyncTaskService(taskDAO)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ start := time.Now()+ var secondBatchAt time.Time+ connector := &connectormock.Connector{+ SyncBatches: []syncerconnector.SyncBatch{+ {Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: start}, {SourceID: "b", UpdatedAt: start}}},+ {Documents: []syncerconnector.SourceDocument{{SourceID: "c", UpdatedAt: start}}},+ },+ OnSyncBatch: func(index int) {+ if index == 1 {+ secondBatchAt = time.Now()+ }+ },+ }+ sink := &fakeSink{delay: 80 * time.Millisecond}+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 2)+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {+ t.Fatalf("execute: %v", err)+ }+ if secondBatchAt.Sub(start) < 70*time.Millisecond {+ t.Fatalf("second batch was read before first batch completed")+ }+}++// TestSyncRunnerProcessesBatchItemsInParallel verifies per-batch item concurrency.+func TestSyncRunnerProcessesBatchItemsInParallel(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ now := time.Now()+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "a", UpdatedAt: now}, {SourceID: "b", UpdatedAt: now}, {SourceID: "c", UpdatedAt: now}}}}}+ sink := &fakeSink{delay: 80 * time.Millisecond}+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 3)+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {+ t.Fatalf("execute: %v", err)+ }+ if sink.maxConcurrent < 2 {+ t.Fatalf("max concurrent items = %d, want >= 2", sink.maxConcurrent)+ }+}++// TestHash128MatchesPythonGolden verifies Python-compatible xxhash128.+func TestHash128MatchesPythonGolden(t *testing.T) {+ cases := map[string]string{+ "connector-1:source-1": "0b66c92e6cefff918067e3c42606cda8",+ "kb-1:connector-1:source-1": "ef785a8871b90d910f42cf31a8155476",+ "": "99aa06d3014798d86001c324468d497f",+ "hello": "b5e9c1ad071b3e7fc779cfaa5e523818",+ }+ for input, want := range cases {+ if got := service.Hash128(input); got != want {+ t.Fatalf("Hash128(%q) = %s, want %s", input, got, want)+ }+ }+}++// TestFingerprintSkipsUnchangedDocument verifies unchanged docs are not fetched or upserted.+func TestFingerprintSkipsUnchangedDocument(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ legacyID := service.Hash128("conn-1:source-1")+ store := fakeStore{ids: map[string]struct{}{legacyID: {}}, fingerprints: map[string]string{legacyID: "fp-1"}}+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Fingerprint: "fp-1", FetchRef: &syncerconnector.FetchReference{Key: "lazy"}, UpdatedAt: time.Now()}}}}}+ sink := &fakeSink{}+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, store, 1)+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {+ t.Fatalf("execute: %v", err)+ }+ if sink.callCount() != 0 {+ t.Fatalf("sink calls = %d, want 0", sink.callCount())+ }+}++// TestAutoParseFlagFlowsToSink verifies connector2kb.auto_parse is preserved.+func TestAutoParseFlagFlowsToSink(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ if err := db.Model(&entity.Connector2Kb{}).Where("connector_id = ? AND kb_id = ?", "conn-1", "kb-1").Update("auto_parse", "0").Error; err != nil {+ t.Fatalf("disable auto_parse: %v", err)+ }+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}+ sink := &fakeSink{}+ coordinator := newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 1)+ taskContext, _ := taskService.GetContext(t.Context(), "task-1")+ if err := coordinator.Execute(t.Context(), taskContext); err != nil {+ t.Fatalf("execute: %v", err)+ }+ if sink.autoParseByDoc["source-1"] {+ t.Fatalf("auto_parse flowed as true, want false")+ }+}++// TestBatchFailureDoesNotAdvanceWaterline verifies failed tasks keep poll_range_end.+func TestBatchFailureDoesNotAdvanceWaterline(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ connector := &connectormock.Connector{SyncBatches: []syncerconnector.SyncBatch{{Documents: []syncerconnector.SourceDocument{{SourceID: "bad", Blob: []byte("x"), UpdatedAt: time.Now()}}}}}+ sink := &fakeSink{errBySourceID: map[string]error{"bad": errors.New("boom")}}+ queue := make(chan TaskEnvelope, 1)+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, 1), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})+ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusFail {+ t.Fatalf("status = %s, want fail", task.Status)+ }+ if task.PollRangeEnd != nil {+ t.Fatalf("poll_range_end advanced on failure")+ }+}++// TestRecoverStaleRunningTasks verifies timeout recovery.+func TestRecoverStaleRunningTasks(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)+ started := time.Now().Add(-2 * time.Hour)+ if err := db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Updates(map[string]any{"status": dao.SyncStatusRunning, "time_started": started}).Error; err != nil {+ t.Fatalf("mark running: %v", err)+ }+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ if err := taskService.RecoverStaleRunning(t.Context(), time.Now()); err != nil {+ t.Fatalf("recover: %v", err)+ }+ var task entity.SyncLogs+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != dao.SyncStatusSchedule {+ t.Fatalf("status = %s, want schedule", task.Status)+ }+}++// TestPruneSnapshotFailureDoesNotDelete verifies incomplete snapshots never delete.+func TestPruneSnapshotFailureDoesNotDelete(t *testing.T) {+ db := setupSyncerDB(t)+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypePrune)+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error+ taskService := service.NewSyncTaskService(dao.NewSyncTaskDAO(db))+ deleter := &fakeDeleter{}+ pruneService := service.NewSyncPruneService(dao.NewSyncPruneSnapshotDAO(db), deleter, fakeStore{ids: map[string]struct{}{"stale": {}}})+ connector := &connectormock.Connector{PruneErrAt: 1, PruneBatches: []syncerconnector.PruneBatch{{Documents: []syncerconnector.SlimDocument{{SourceID: "keep"}}}}}+ queue := make(chan TaskEnvelope, 1)+ worker := NewTaskWorker(queue, taskService, newCoordinator(taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), &fakeSink{}, pruneService, fakeStore{}, 1), NewConnectorLock())+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})+ if len(deleter.deleted) != 0 {+ t.Fatalf("deleted %v despite incomplete snapshot", deleter.deleted)+ }+ var count int64+ if err := db.Model(&dao.SyncPruneSnapshot{}).Where("task_id = ?", "task-1").Count(&count).Error; err != nil {+ t.Fatalf("count snapshot: %v", err)+ }+ if count != 0 {+ t.Fatalf("snapshot rows = %d, want cleared", count)+ }+}++// TestPruneRetainIDsIncludeLegacyAndNew verifies PRUNE retains both ID schemes.+func TestPruneRetainIDsIncludeLegacyAndNew(t *testing.T) {+ retain := service.RetainDocumentIDs("kb-1", "conn-1", []string{"source-1"})+ legacy := service.Hash128("conn-1:source-1")+ next := service.Hash128("kb-1:conn-1:source-1")+ if _, ok := retain[legacy]; !ok {+ t.Fatalf("legacy ID missing from retain set")+ }+ if _, ok := retain[next]; !ok {+ t.Fatalf("new ID missing from retain set")+ }+}++// TestMockSessionEOF documents the mock session EOF contract.+func TestMockSessionEOF(t *testing.T) {+ session, _ := (&connectormock.Connector{}).OpenSync(t.Context(), syncerconnector.SyncRequest{})+ _, err := session.NextBatch(t.Context())+ if !errors.Is(err, io.EOF) {+ t.Fatalf("err = %v, want io.EOF", err)+ }+}diff --git a/internal/syncer/task_coordinator.go b/internal/syncer/task_coordinator.gonew file mode 100644index 0000000000..f129ea5da2--- /dev/null+++ b/internal/syncer/task_coordinator.go@@ -0,0 +1,87 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "fmt"+ "ragflow/internal/service"+ syncerconnector "ragflow/internal/syncer/connector"+ "time"+)++// ConnectorRegistry opens registered connectors by source.+type ConnectorRegistry interface {+ // Open creates a connector for a task context.+ Open(ctx context.Context, taskContext any) (syncerconnector.Connector, error)+}++// TaskCoordinatorConfig controls per-task document processing.+type TaskCoordinatorConfig struct {+ PerTaskItemConcurrency int+ ItemRetryCount int+ ItemRetryBaseDelay time.Duration+}++// TaskCoordinator owns one task execution window.+type TaskCoordinator struct {+ config TaskCoordinatorConfig+ taskService *service.SyncTaskService+ registry ConnectorRegistry+ sink service.DocumentSink+ pruneService *service.SyncPruneService+ idResolver *service.DocumentIDResolver+ globalItems chan struct{}+}++// NewTaskCoordinator creates a coordinator for one claimed task at a time.+func NewTaskCoordinator(config TaskCoordinatorConfig, taskService *service.SyncTaskService, registry ConnectorRegistry, sink service.DocumentSink, pruneService *service.SyncPruneService, idResolver *service.DocumentIDResolver, globalItems chan struct{}) *TaskCoordinator {+ if config.PerTaskItemConcurrency <= 0 {+ config.PerTaskItemConcurrency = 1+ }++ if config.ItemRetryCount <= 0 {+ config.ItemRetryCount = 1+ }++ if config.ItemRetryBaseDelay <= 0 {+ config.ItemRetryBaseDelay = time.Second+ }++ return &TaskCoordinator{config: config, taskService: taskService, registry: registry, sink: sink, pruneService: pruneService, idResolver: idResolver, globalItems: globalItems}+}++// Execute dispatches a sync_logs task by task type.+func (c *TaskCoordinator) Execute(ctx context.Context, taskContext service.SyncTaskContext) error {+ connector, err := c.registry.Open(ctx, taskContext)+ if err != nil {+ return err+ }+ if err = connector.Validate(ctx); err != nil {+ return err+ }+ switch taskContext.Task.TaskType {+ case service.TaskTypeSync:+ runner := NewSyncRunner(c.config, c.taskService, c.sink, c.idResolver, c.globalItems)+ return runner.Run(ctx, taskContext, connector)+ case service.TaskTypePrune:+ runner := NewPruneRunner(c.taskService, c.pruneService)+ return runner.Run(ctx, taskContext, connector)+ default:+ return fmt.Errorf("unsupported sync task type %q", taskContext.Task.TaskType)+ }+}diff --git a/internal/syncer/task_worker.go b/internal/syncer/task_worker.gonew file mode 100644index 0000000000..230ee9bbd3--- /dev/null+++ b/internal/syncer/task_worker.go@@ -0,0 +1,89 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package syncer++import (+ "context"+ "fmt"+ "ragflow/internal/service"+ "sync"+)++// TaskWorker consumes claimed task envelopes and runs coordinators.+type TaskWorker struct {+ queue <-chan TaskEnvelope+ taskService *service.SyncTaskService+ coordinator *TaskCoordinator+ locker ConnectorLocker+}++// NewTaskWorker creates a bounded task worker pool.+func NewTaskWorker(queue <-chan TaskEnvelope, taskService *service.SyncTaskService, coordinator *TaskCoordinator, locker ConnectorLocker) *TaskWorker {+ return &TaskWorker{queue: queue, taskService: taskService, coordinator: coordinator, locker: locker}+}++// Run starts worker goroutines and blocks until context cancellation.+func (w *TaskWorker) Run(ctx context.Context, concurrency int) {+ var wg sync.WaitGroup+ for i := 0; i < concurrency; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ w.loop(ctx)+ }()+ }+ <-ctx.Done()+ wg.Wait()+}++// loop handles queue messages until cancellation.+func (w *TaskWorker) loop(ctx context.Context) {+ for {+ select {+ case <-ctx.Done():+ return+ case envelope := <-w.queue:+ w.handle(ctx, envelope)+ }+ }+}++// handle loads a claimed task and executes it under the connector lock.+func (w *TaskWorker) handle(ctx context.Context, envelope TaskEnvelope) {+ taskContext, err := w.taskService.GetContext(ctx, envelope.TaskID)+ if err != nil {+ if ctx.Err() != nil {+ _ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), envelope.TaskID)+ return+ }+ _ = w.taskService.Fail(ctx, envelope.TaskID, "", err)+ return+ }+ if !w.locker.TryLock(taskContext.Connector.ID) {+ _ = w.taskService.RescheduleClaimed(ctx, taskContext.Task.ID)+ return+ }+ defer w.locker.Unlock(taskContext.Connector.ID)++ if err = w.coordinator.Execute(ctx, taskContext); err != nil {+ if ctx.Err() != nil {+ _ = w.taskService.RescheduleClaimed(context.WithoutCancel(ctx), taskContext.Task.ID)+ return+ }+ _ = w.taskService.Fail(ctx, taskContext.Task.ID, taskContext.Connector.ID, fmt.Errorf("sync task failed: %w", err))+ }+}diff --git a/internal/utility/upload.go b/internal/utility/upload.goindex 030ef1c4e6..585dc668b9 100644--- a/internal/utility/upload.go+++ b/internal/utility/upload.go@@ -39,19 +39,31 @@ var (// timeouts, and a hard size cap that rejects (rather than truncates) oversized// bodies.func FetchRemoteFileSafely(ctx context.Context, rawURL string, maxSize int64) ([]byte, http.Header, string, error) {+ return FetchRemoteFileSafelyWithTimeout(ctx, rawURL, maxSize, 10*time.Second)+}++// FetchRemoteFileSafelyWithTimeout downloads rawURL with a caller-selected timeout.+func FetchRemoteFileSafelyWithTimeout(ctx context.Context, rawURL string, maxSize int64, timeout time.Duration) ([]byte, http.Header, string, error) {+ if timeout <= 0 {+ timeout = 10 * time.Second+ }currentURL := rawURLfor redirects := 0; redirects < 10; redirects++ {hostname, resolvedIP, err := AssertURLSafe(currentURL)if err != nil {return nil, nil, "", err}- client := PinnedHTTPClient(hostname, resolvedIP, 10*time.Second)+ client := PinnedHTTPClient(hostname, resolvedIP, timeout)client.CheckRedirect = func(req *http.Request, via []*http.Request) error {return http.ErrUseLastResponse}// codeql[go/request-forgery] False positive: the loop above- resp, err := client.Get(currentURL) // #nosec G107+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil)+ if err != nil {+ return nil, nil, "", fmt.Errorf("failed to create request: %w", err)+ }+ resp, err := client.Do(req) // `#nosec` G107if err != nil {return nil, nil, "", fmt.Errorf("failed to fetch URL: %w", err)}