RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): fail closed on non-positive connector refresh freq (#20871)
jay77721db125c7188b7
diff --git a/internal/handler/connector.go b/internal/handler/connector.goindex 163c7d74e8..9f46ba6cd0 100644--- a/internal/handler/connector.go+++ b/internal/handler/connector.go@@ -337,6 +337,10 @@ func (h *ConnectorHandler) CreateConnector(c *gin.Context) {connector, err := h.connectorService.CreateConnector(ctx, user.ID, &req)if err != nil {+ if errors.Is(err, service.ErrConnectorInvalidFreq) {+ common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())+ return+ }common.ResponseWithHttpCodeData(c, http.StatusInternalServerError, common.CodeServerError, nil, err.Error())return}diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.goindex 4ff6d15f9f..acc5a7bc92 100644--- a/internal/handler/connector_test.go+++ b/internal/handler/connector_test.go@@ -229,6 +229,49 @@ func TestConnectorHandlerGetConnector(t *testing.T) {}}+func TestConnectorHandlerCreateConnectorMapsInvalidFreq(t *testing.T) {+ gin.SetMode(gin.TestMode)++ tests := []struct {+ name string+ err error+ wantCode common.ErrorCode+ }{+ {+ name: "invalid frequency is a client error",+ err: fmt.Errorf("%w: refresh_freq got 0", service.ErrConnectorInvalidFreq),+ wantCode: common.CodeDataError,+ },+ {+ name: "internal failure stays a server error",+ err: fmt.Errorf("boom"),+ wantCode: common.CodeServerError,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ h := &ConnectorHandler{connectorService: fakeConnectorService{err: tt.err}}+ resp := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(resp)+ c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/connectors",+ strings.NewReader(`{"name":"c","source":"rss","config":{}}`))+ c.Request.Header.Set("Content-Type", "application/json")+ c.Set("user", &entity.User{ID: "user-1"})++ h.CreateConnector(c)++ var body map[string]interface{}+ if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {+ t.Fatalf("unmarshal response: %v", err)+ }+ if body["code"] != float64(tt.wantCode) {+ t.Fatalf("code=%v want=%v body=%v", body["code"], tt.wantCode, body)+ }+ })+ }+}+func TestConnectorHandlerStartBoxWebOAuth(t *testing.T) {gin.SetMode(gin.TestMode)diff --git a/internal/service/connector.go b/internal/service/connector.goindex 33e78c3598..167f5764cd 100644--- a/internal/service/connector.go+++ b/internal/service/connector.go@@ -86,6 +86,9 @@ var (ErrConnectorSourceNotImplemented = errors.New("connector source is not implemented")// ErrConnectorInternal is a generic, safe-to-expose internal failure.ErrConnectorInternal = errors.New("Internal server error")+ // ErrConnectorInvalidFreq is returned when a connector frequency is not+ // positive. Handlers map it to a client error; callers match with errors.Is.+ ErrConnectorInvalidFreq = errors.New("connector frequency must be positive"))// ConnectorService connector service@@ -275,6 +278,12 @@ func (s *ConnectorService) cancelConnectorTasks(ctx context.Context, connectorID// CreateConnector creates a connector owned by the current user.func (s *ConnectorService) CreateConnector(ctx context.Context, userID string, req *CreateConnectorRequest) (*entity.Connector, error) {+ if req.RefreshFreq != nil && *req.RefreshFreq <= 0 {+ return nil, fmt.Errorf("%w: refresh_freq got %d", ErrConnectorInvalidFreq, *req.RefreshFreq)+ }+ if req.PruneFreq != nil && *req.PruneFreq <= 0 {+ return nil, fmt.Errorf("%w: prune_freq got %d", ErrConnectorInvalidFreq, *req.PruneFreq)+ }refreshFreq := int64(defaultConnectorFreq)if req.RefreshFreq != nil {refreshFreq = *req.RefreshFreq@@ -974,9 +983,15 @@ func (s *ConnectorService) UpdateConnector(ctx context.Context, connectorID, useupdates := map[string]interface{}{}if req != nil {if req.PruneFreq != nil {+ if *req.PruneFreq <= 0 {+ return nil, common.CodeDataError, fmt.Errorf("%w: prune_freq got %d", ErrConnectorInvalidFreq, *req.PruneFreq)+ }updates["prune_freq"] = *req.PruneFreq}if req.RefreshFreq != nil {+ if *req.RefreshFreq <= 0 {+ return nil, common.CodeDataError, fmt.Errorf("%w: refresh_freq got %d", ErrConnectorInvalidFreq, *req.RefreshFreq)+ }updates["refresh_freq"] = *req.RefreshFreq}if req.Config != nil {diff --git a/internal/service/connector_freq_test.go b/internal/service/connector_freq_test.gonew file mode 100644index 0000000000..fae7c9084e--- /dev/null+++ b/internal/service/connector_freq_test.go@@ -0,0 +1,72 @@+package service++import (+ "errors"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity"+)++func setupFreqValidationConnector(t *testing.T) {+ t.Helper()+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ if err := db.AutoMigrate(&entity.Connector{}); err != nil {+ t.Fatalf("migrate connector tables: %v", err)+ }+ if err := db.Create(&entity.Connector{+ ID: "conn-freq",+ TenantID: "user-1",+ Name: "conn-freq",+ Source: "rss",+ InputType: "poll",+ Config: entity.JSONMap{},+ Status: string(entity.TaskStatusCancel),+ RefreshFreq: 5,+ PruneFreq: 5,+ TimeoutSecs: 60,+ }).Error; err != nil {+ t.Fatalf("insert connector: %v", err)+ }+}++func TestCreateConnectorRejectsNonPositiveFreq(t *testing.T) {+ setupFreqValidationConnector(t)+ svc := NewConnectorService()++ for name, req := range map[string]*CreateConnectorRequest{+ "refresh zero": {Name: "c", Source: "rss", Config: entity.JSONMap{}, RefreshFreq: int64Ptr(0)},+ "refresh negative": {Name: "c", Source: "rss", Config: entity.JSONMap{}, RefreshFreq: int64Ptr(-3)},+ "prune zero": {Name: "c", Source: "rss", Config: entity.JSONMap{}, PruneFreq: int64Ptr(0)},+ "prune negative": {Name: "c", Source: "rss", Config: entity.JSONMap{}, PruneFreq: int64Ptr(-1)},+ } {+ if _, err := svc.CreateConnector(t.Context(), "user-1", req); err == nil {+ t.Fatalf("%s: CreateConnector succeeded, want frequency error", name)+ } else if !errors.Is(err, ErrConnectorInvalidFreq) {+ t.Fatalf("%s: error = %v, want ErrConnectorInvalidFreq", name, err)+ }+ }+}++func TestUpdateConnectorRejectsNonPositiveFreq(t *testing.T) {+ setupFreqValidationConnector(t)+ svc := NewConnectorService()++ for name, req := range map[string]*UpdateConnectorRequest{+ "refresh zero": {RefreshFreq: int64Ptr(0)},+ "refresh negative": {RefreshFreq: int64Ptr(-3)},+ "prune zero": {PruneFreq: int64Ptr(0)},+ "prune negative": {PruneFreq: int64Ptr(-1)},+ } {+ _, code, err := svc.UpdateConnector(t.Context(), "conn-freq", "user-1", req)+ if err == nil {+ t.Fatalf("%s: UpdateConnector succeeded, want frequency error", name)+ }+ if code != common.CodeDataError {+ t.Fatalf("%s: code = %v, want CodeDataError", name, code)+ }+ }+}++func int64Ptr(v int64) *int64 { return &v }diff --git a/internal/syncer/scheduler.go b/internal/syncer/scheduler.goindex 97078f3790..6f48ec2216 100644--- a/internal/syncer/scheduler.go+++ b/internal/syncer/scheduler.go@@ -253,6 +253,10 @@ func (s *Scheduler) stopTimers() {}// taskDelay reports the delay before publication and whether the task must be scheduled at all.+// A non-positive frequency disables periodic rescheduling (fail-closed):+// the task runs at most once (when never run before) instead of being+// republished with no interval, which would hot-loop sync workers, NATS,+// and the upstream source.func (s *Scheduler) taskDelay(task dao.ScheduledSyncTask, now time.Time) (time.Duration, bool) {freq := int64(0)switch task.TaskType {@@ -264,8 +268,16 @@ func (s *Scheduler) taskDelay(task dao.ScheduledSyncTask, now time.Time) (time.D}freq = task.ConnectorPruneFreq}- if freq <= 0 || task.UpdateDate == nil {+ if task.UpdateDate == nil {return 0, true}+ if freq <= 0 {+ common.Warn("syncer task not scheduled: non-positive frequency",+ zap.String("task_id", task.ID),+ zap.String("connector_id", task.ConnectorID),+ zap.String("task_type", task.TaskType),+ zap.Int64("frequency_minutes", freq))+ return 0, false+ }return task.UpdateDate.Add(time.Duration(freq) * time.Minute).Sub(now), true}diff --git a/internal/syncer/scheduler_taskdelay_test.go b/internal/syncer/scheduler_taskdelay_test.gonew file mode 100644index 0000000000..8c681264df--- /dev/null+++ b/internal/syncer/scheduler_taskdelay_test.go@@ -0,0 +1,81 @@+package syncer++import (+ "testing"+ "time"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+)++func testTaskDelayTask(taskType string, freq int64, updateDate *time.Time) dao.ScheduledSyncTask {+ task := dao.ScheduledSyncTask{ConnectorConfig: entity.JSONMap{}}+ task.TaskType = taskType+ task.ConnectorRefreshFreq = freq+ task.ConnectorPruneFreq = freq+ task.UpdateDate = updateDate+ return task+}++// TestTaskDelayNonPositiveFreqDoesNotReschedule pins the hot-loop fix: a+// task whose connector frequency is zero or negative must not be+// republished once it has run before. Only a never-run task runs once.+func TestTaskDelayNonPositiveFreqDoesNotReschedule(t *testing.T) {+ scheduler := NewScheduler(nil, nil)+ past := time.Now().Add(-time.Hour)++ for _, freq := range []int64{0, -5} {+ task := testTaskDelayTask(dao.TaskTypeSync, freq, &past)+ if delay, schedule := scheduler.taskDelay(task, time.Now()); schedule {+ t.Fatalf("sync freq=%d with past run: schedule=true (delay=%v), want false", freq, delay)+ }++ prune := testTaskDelayTask(dao.TaskTypePrune, freq, &past)+ prune.ConnectorConfig = entity.JSONMap{"sync_deleted_files": true}+ if delay, schedule := scheduler.taskDelay(prune, time.Now()); schedule {+ t.Fatalf("prune freq=%d with past run: schedule=true (delay=%v), want false", freq, delay)+ }+ }+}++// TestTaskDelayNeverRunPublishesOnce ensures a never-run task still runs+// exactly once, even with a non-positive frequency.+func TestTaskDelayNeverRunPublishesOnce(t *testing.T) {+ scheduler := NewScheduler(nil, nil)++ task := testTaskDelayTask(dao.TaskTypeSync, 0, nil)+ delay, schedule := scheduler.taskDelay(task, time.Now())+ if !schedule {+ t.Fatal("never-run sync task: schedule=false, want true")+ }+ if delay != 0 {+ t.Fatalf("never-run sync task: delay=%v, want 0", delay)+ }+}++// TestTaskDelayPositiveFreqSchedulesByUpdateDate guards the normal path:+// overdue tasks are due now, recently run tasks wait out their interval.+func TestTaskDelayPositiveFreqSchedulesByUpdateDate(t *testing.T) {+ scheduler := NewScheduler(nil, nil)+ now := time.Now()++ overdue := now.Add(-10 * time.Minute)+ task := testTaskDelayTask(dao.TaskTypeSync, 5, &overdue)+ delay, schedule := scheduler.taskDelay(task, now)+ if !schedule {+ t.Fatal("overdue sync task: schedule=false, want true")+ }+ if delay > 0 {+ t.Fatalf("overdue sync task: delay=%v, want <= 0", delay)+ }++ recent := now+ waiting := testTaskDelayTask(dao.TaskTypeSync, 60, &recent)+ delay, schedule = scheduler.taskDelay(waiting, now)+ if !schedule {+ t.Fatal("recent sync task: schedule=false, want true")+ }+ if delay <= 50*time.Minute {+ t.Fatalf("recent sync task: delay=%v, want > 50m", delay)+ }+}diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.goindex 78d64cff62..fba51d3f50 100644--- a/internal/syncer/syncer_test.go+++ b/internal/syncer/syncer_test.go@@ -243,14 +243,18 @@ func insertTaskContext(t *testing.T, db *gorm.DB, connectorID, kbID, taskID, tas}if existingConnector == 0 {if err := db.Create(&entity.Connector{- ID: connectorID,- TenantID: "tenant-1",- Name: connectorID,- Source: "mock",- InputType: "poll",- Config: entity.JSONMap{"sync_deleted_files": true},- RefreshFreq: 0,- PruneFreq: 0,+ ID: connectorID,+ TenantID: "tenant-1",+ Name: connectorID,+ Source: "mock",+ InputType: "poll",+ Config: entity.JSONMap{"sync_deleted_files": true},+ // Keep frequencies positive: taskDelay treats freq <= 0 as+ // "no periodic reschedule" (fail-closed against hot-loop+ // republishing), so a zero fixture would silently suppress+ // every startup-publish expectation below.+ RefreshFreq: 5,+ PruneFreq: 5,TimeoutSecs: 60,Status: dao.SyncStatusSchedule,BaseModel: entity.BaseModel{UpdateDate: &now, UpdateTime: &ts},