Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): fail closed on non-positive connector refresh freq (#20871)

jay77721db125c7188b7
diff --git a/internal/handler/connector.go b/internal/handler/connector.go
index 163c7d74e8..9f46ba6cd0 100644
--- a/internal/handler/connector.go
+++ b/internal/handler/connector.go
@@ -337,6 +337,10 @@ func (h *ConnectorHandler) CreateConnector(c *gin.Context) {
connector, err := h.connectorService.CreateConnector(ctx, user.ID, &req)
if err != nil {
+ if errors.Is(err, service.ErrConnectorInvalidFreq) {
+ common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
+ return
+ }
common.ResponseWithHttpCodeData(c, http.StatusInternalServerError, common.CodeServerError, nil, err.Error())
return
}
diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.go
index 4ff6d15f9f..acc5a7bc92 100644
--- a/internal/handler/connector_test.go
+++ b/internal/handler/connector_test.go
@@ -229,6 +229,49 @@ func TestConnectorHandlerGetConnector(t *testing.T) {
}
}
+func TestConnectorHandlerCreateConnectorMapsInvalidFreq(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ tests := []struct {
+ name string
+ err error
+ wantCode common.ErrorCode
+ }{
+ {
+ name: "invalid frequency is a client error",
+ err: fmt.Errorf("%w: refresh_freq got 0", service.ErrConnectorInvalidFreq),
+ wantCode: common.CodeDataError,
+ },
+ {
+ name: "internal failure stays a server error",
+ err: fmt.Errorf("boom"),
+ wantCode: common.CodeServerError,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ h := &ConnectorHandler{connectorService: fakeConnectorService{err: tt.err}}
+ resp := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(resp)
+ c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/connectors",
+ strings.NewReader(`{"name":"c","source":"rss","config":{}}`))
+ c.Request.Header.Set("Content-Type", "application/json")
+ c.Set("user", &entity.User{ID: "user-1"})
+
+ h.CreateConnector(c)
+
+ var body map[string]interface{}
+ if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if body["code"] != float64(tt.wantCode) {
+ t.Fatalf("code=%v want=%v body=%v", body["code"], tt.wantCode, body)
+ }
+ })
+ }
+}
+
func TestConnectorHandlerStartBoxWebOAuth(t *testing.T) {
gin.SetMode(gin.TestMode)
diff --git a/internal/service/connector.go b/internal/service/connector.go
index 33e78c3598..167f5764cd 100644
--- a/internal/service/connector.go
+++ b/internal/service/connector.go
@@ -86,6 +86,9 @@ var (
ErrConnectorSourceNotImplemented = errors.New("connector source is not implemented")
// ErrConnectorInternal is a generic, safe-to-expose internal failure.
ErrConnectorInternal = errors.New("Internal server error")
+ // ErrConnectorInvalidFreq is returned when a connector frequency is not
+ // positive. Handlers map it to a client error; callers match with errors.Is.
+ ErrConnectorInvalidFreq = errors.New("connector frequency must be positive")
)
// ConnectorService connector service
@@ -275,6 +278,12 @@ func (s *ConnectorService) cancelConnectorTasks(ctx context.Context, connectorID
// CreateConnector creates a connector owned by the current user.
func (s *ConnectorService) CreateConnector(ctx context.Context, userID string, req *CreateConnectorRequest) (*entity.Connector, error) {
+ if req.RefreshFreq != nil && *req.RefreshFreq <= 0 {
+ return nil, fmt.Errorf("%w: refresh_freq got %d", ErrConnectorInvalidFreq, *req.RefreshFreq)
+ }
+ if req.PruneFreq != nil && *req.PruneFreq <= 0 {
+ return nil, fmt.Errorf("%w: prune_freq got %d", ErrConnectorInvalidFreq, *req.PruneFreq)
+ }
refreshFreq := int64(defaultConnectorFreq)
if req.RefreshFreq != nil {
refreshFreq = *req.RefreshFreq
@@ -974,9 +983,15 @@ func (s *ConnectorService) UpdateConnector(ctx context.Context, connectorID, use
updates := map[string]interface{}{}
if req != nil {
if req.PruneFreq != nil {
+ if *req.PruneFreq <= 0 {
+ return nil, common.CodeDataError, fmt.Errorf("%w: prune_freq got %d", ErrConnectorInvalidFreq, *req.PruneFreq)
+ }
updates["prune_freq"] = *req.PruneFreq
}
if req.RefreshFreq != nil {
+ if *req.RefreshFreq <= 0 {
+ return nil, common.CodeDataError, fmt.Errorf("%w: refresh_freq got %d", ErrConnectorInvalidFreq, *req.RefreshFreq)
+ }
updates["refresh_freq"] = *req.RefreshFreq
}
if req.Config != nil {
diff --git a/internal/service/connector_freq_test.go b/internal/service/connector_freq_test.go
new file mode 100644
index 0000000000..fae7c9084e
--- /dev/null
+++ b/internal/service/connector_freq_test.go
@@ -0,0 +1,72 @@
+package service
+
+import (
+ "errors"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+)
+
+func setupFreqValidationConnector(t *testing.T) {
+ t.Helper()
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+ if err := db.Create(&entity.Connector{
+ ID: "conn-freq",
+ TenantID: "user-1",
+ Name: "conn-freq",
+ Source: "rss",
+ InputType: "poll",
+ Config: entity.JSONMap{},
+ Status: string(entity.TaskStatusCancel),
+ RefreshFreq: 5,
+ PruneFreq: 5,
+ TimeoutSecs: 60,
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+}
+
+func TestCreateConnectorRejectsNonPositiveFreq(t *testing.T) {
+ setupFreqValidationConnector(t)
+ svc := NewConnectorService()
+
+ for name, req := range map[string]*CreateConnectorRequest{
+ "refresh zero": {Name: "c", Source: "rss", Config: entity.JSONMap{}, RefreshFreq: int64Ptr(0)},
+ "refresh negative": {Name: "c", Source: "rss", Config: entity.JSONMap{}, RefreshFreq: int64Ptr(-3)},
+ "prune zero": {Name: "c", Source: "rss", Config: entity.JSONMap{}, PruneFreq: int64Ptr(0)},
+ "prune negative": {Name: "c", Source: "rss", Config: entity.JSONMap{}, PruneFreq: int64Ptr(-1)},
+ } {
+ if _, err := svc.CreateConnector(t.Context(), "user-1", req); err == nil {
+ t.Fatalf("%s: CreateConnector succeeded, want frequency error", name)
+ } else if !errors.Is(err, ErrConnectorInvalidFreq) {
+ t.Fatalf("%s: error = %v, want ErrConnectorInvalidFreq", name, err)
+ }
+ }
+}
+
+func TestUpdateConnectorRejectsNonPositiveFreq(t *testing.T) {
+ setupFreqValidationConnector(t)
+ svc := NewConnectorService()
+
+ for name, req := range map[string]*UpdateConnectorRequest{
+ "refresh zero": {RefreshFreq: int64Ptr(0)},
+ "refresh negative": {RefreshFreq: int64Ptr(-3)},
+ "prune zero": {PruneFreq: int64Ptr(0)},
+ "prune negative": {PruneFreq: int64Ptr(-1)},
+ } {
+ _, code, err := svc.UpdateConnector(t.Context(), "conn-freq", "user-1", req)
+ if err == nil {
+ t.Fatalf("%s: UpdateConnector succeeded, want frequency error", name)
+ }
+ if code != common.CodeDataError {
+ t.Fatalf("%s: code = %v, want CodeDataError", name, code)
+ }
+ }
+}
+
+func int64Ptr(v int64) *int64 { return &v }
diff --git a/internal/syncer/scheduler.go b/internal/syncer/scheduler.go
index 97078f3790..6f48ec2216 100644
--- a/internal/syncer/scheduler.go
+++ b/internal/syncer/scheduler.go
@@ -253,6 +253,10 @@ func (s *Scheduler) stopTimers() {
}
// taskDelay reports the delay before publication and whether the task must be scheduled at all.
+// A non-positive frequency disables periodic rescheduling (fail-closed):
+// the task runs at most once (when never run before) instead of being
+// republished with no interval, which would hot-loop sync workers, NATS,
+// and the upstream source.
func (s *Scheduler) taskDelay(task dao.ScheduledSyncTask, now time.Time) (time.Duration, bool) {
freq := int64(0)
switch task.TaskType {
@@ -264,8 +268,16 @@ func (s *Scheduler) taskDelay(task dao.ScheduledSyncTask, now time.Time) (time.D
}
freq = task.ConnectorPruneFreq
}
- if freq <= 0 || task.UpdateDate == nil {
+ if task.UpdateDate == nil {
return 0, true
}
+ if freq <= 0 {
+ common.Warn("syncer task not scheduled: non-positive frequency",
+ zap.String("task_id", task.ID),
+ zap.String("connector_id", task.ConnectorID),
+ zap.String("task_type", task.TaskType),
+ zap.Int64("frequency_minutes", freq))
+ return 0, false
+ }
return task.UpdateDate.Add(time.Duration(freq) * time.Minute).Sub(now), true
}
diff --git a/internal/syncer/scheduler_taskdelay_test.go b/internal/syncer/scheduler_taskdelay_test.go
new file mode 100644
index 0000000000..8c681264df
--- /dev/null
+++ b/internal/syncer/scheduler_taskdelay_test.go
@@ -0,0 +1,81 @@
+package syncer
+
+import (
+ "testing"
+ "time"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+)
+
+func testTaskDelayTask(taskType string, freq int64, updateDate *time.Time) dao.ScheduledSyncTask {
+ task := dao.ScheduledSyncTask{ConnectorConfig: entity.JSONMap{}}
+ task.TaskType = taskType
+ task.ConnectorRefreshFreq = freq
+ task.ConnectorPruneFreq = freq
+ task.UpdateDate = updateDate
+ return task
+}
+
+// TestTaskDelayNonPositiveFreqDoesNotReschedule pins the hot-loop fix: a
+// task whose connector frequency is zero or negative must not be
+// republished once it has run before. Only a never-run task runs once.
+func TestTaskDelayNonPositiveFreqDoesNotReschedule(t *testing.T) {
+ scheduler := NewScheduler(nil, nil)
+ past := time.Now().Add(-time.Hour)
+
+ for _, freq := range []int64{0, -5} {
+ task := testTaskDelayTask(dao.TaskTypeSync, freq, &past)
+ if delay, schedule := scheduler.taskDelay(task, time.Now()); schedule {
+ t.Fatalf("sync freq=%d with past run: schedule=true (delay=%v), want false", freq, delay)
+ }
+
+ prune := testTaskDelayTask(dao.TaskTypePrune, freq, &past)
+ prune.ConnectorConfig = entity.JSONMap{"sync_deleted_files": true}
+ if delay, schedule := scheduler.taskDelay(prune, time.Now()); schedule {
+ t.Fatalf("prune freq=%d with past run: schedule=true (delay=%v), want false", freq, delay)
+ }
+ }
+}
+
+// TestTaskDelayNeverRunPublishesOnce ensures a never-run task still runs
+// exactly once, even with a non-positive frequency.
+func TestTaskDelayNeverRunPublishesOnce(t *testing.T) {
+ scheduler := NewScheduler(nil, nil)
+
+ task := testTaskDelayTask(dao.TaskTypeSync, 0, nil)
+ delay, schedule := scheduler.taskDelay(task, time.Now())
+ if !schedule {
+ t.Fatal("never-run sync task: schedule=false, want true")
+ }
+ if delay != 0 {
+ t.Fatalf("never-run sync task: delay=%v, want 0", delay)
+ }
+}
+
+// TestTaskDelayPositiveFreqSchedulesByUpdateDate guards the normal path:
+// overdue tasks are due now, recently run tasks wait out their interval.
+func TestTaskDelayPositiveFreqSchedulesByUpdateDate(t *testing.T) {
+ scheduler := NewScheduler(nil, nil)
+ now := time.Now()
+
+ overdue := now.Add(-10 * time.Minute)
+ task := testTaskDelayTask(dao.TaskTypeSync, 5, &overdue)
+ delay, schedule := scheduler.taskDelay(task, now)
+ if !schedule {
+ t.Fatal("overdue sync task: schedule=false, want true")
+ }
+ if delay > 0 {
+ t.Fatalf("overdue sync task: delay=%v, want <= 0", delay)
+ }
+
+ recent := now
+ waiting := testTaskDelayTask(dao.TaskTypeSync, 60, &recent)
+ delay, schedule = scheduler.taskDelay(waiting, now)
+ if !schedule {
+ t.Fatal("recent sync task: schedule=false, want true")
+ }
+ if delay <= 50*time.Minute {
+ t.Fatalf("recent sync task: delay=%v, want > 50m", delay)
+ }
+}
diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go
index 78d64cff62..fba51d3f50 100644
--- a/internal/syncer/syncer_test.go
+++ b/internal/syncer/syncer_test.go
@@ -243,14 +243,18 @@ func insertTaskContext(t *testing.T, db *gorm.DB, connectorID, kbID, taskID, tas
}
if existingConnector == 0 {
if err := db.Create(&entity.Connector{
- ID: connectorID,
- TenantID: "tenant-1",
- Name: connectorID,
- Source: "mock",
- InputType: "poll",
- Config: entity.JSONMap{"sync_deleted_files": true},
- RefreshFreq: 0,
- PruneFreq: 0,
+ ID: connectorID,
+ TenantID: "tenant-1",
+ Name: connectorID,
+ Source: "mock",
+ InputType: "poll",
+ Config: entity.JSONMap{"sync_deleted_files": true},
+ // Keep frequencies positive: taskDelay treats freq <= 0 as
+ // "no periodic reschedule" (fail-closed against hot-loop
+ // republishing), so a zero fixture would silently suppress
+ // every startup-publish expectation below.
+ RefreshFreq: 5,
+ PruneFreq: 5,
TimeoutSecs: 60,
Status: dao.SyncStatusSchedule,
BaseModel: entity.BaseModel{UpdateDate: &now, UpdateTime: &ts},