Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): validate cron schedules and persist manual-only mode (#3653)

Perfecto00e6366fd27a
diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go
index c3b2f58e4..8f2cabbe0 100644
--- a/internal/application/repository/datasource_repo.go
+++ b/internal/application/repository/datasource_repo.go
@@ -90,10 +90,13 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource)
return err
}
// GORM Updates(struct) deliberately skips zero values, which would make
- // a user-selected sync_deletions=false impossible to persist.
+ // a user-selected sync_deletions=false or manual-only schedule impossible to persist.
return tx.Model(&types.DataSource{}).
Where("id = ?", ds.ID).
- UpdateColumn("sync_deletions", ds.SyncDeletions).Error
+ UpdateColumns(map[string]interface{}{
+ "sync_deletions": ds.SyncDeletions,
+ "sync_schedule": ds.SyncSchedule,
+ }).Error
})
}
diff --git a/internal/application/service/datasource_schedule_test.go b/internal/application/service/datasource_schedule_test.go
new file mode 100644
index 000000000..0320bd3f3
--- /dev/null
+++ b/internal/application/service/datasource_schedule_test.go
@@ -0,0 +1,133 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDataSourceResumeRejectsInvalidSchedule(t *testing.T) {
+ f := newSQLiteDataSourceDeleteFixture(t)
+ f.scheduler.Remove(f.ds.ID)
+ f.ds.Status = types.DataSourceStatusPaused
+ f.ds.SyncSchedule = "not-a-cron"
+ require.NoError(t, f.dsRepo.Update(context.Background(), f.ds))
+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
+ err := svc.ResumeDataSource(context.Background(), f.ds.ID)
+ saved, readErr := svc.GetDataSource(context.Background(), f.ds.ID)
+ require.NoError(t, readErr)
+ t.Logf("resume error=%v; status=%s; entries=%d", err, saved.Status, f.scheduler.EntryCount())
+ require.ErrorContains(t, err, "invalid cron expression")
+ require.Equal(t, types.DataSourceStatusPaused, saved.Status)
+ require.Zero(t, f.scheduler.EntryCount())
+}
+
+func TestDataSourceUpdateRejectsInvalidSchedule(t *testing.T) {
+ f := newSQLiteDataSourceDeleteFixture(t)
+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
+ updated := *f.ds
+ updated.SyncSchedule = "not-a-cron"
+ _, err := svc.UpdateDataSource(context.Background(), &updated)
+ require.ErrorContains(t, err, "invalid cron expression")
+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
+ require.NoError(t, err)
+ require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)
+ require.Equal(t, 1, f.scheduler.EntryCount())
+}
+
+func TestDataSourceCreateRejectsInvalidSchedule(t *testing.T) {
+ f := newSQLiteDataSourceDeleteFixture(t)
+ registry := datasource.NewConnectorRegistry()
+ require.NoError(t, registry.Register(deletedItemConnector{}))
+ svc := &DataSourceService{
+ dsRepo: f.dsRepo, scheduler: f.scheduler, connectorRegistry: registry,
+ kbService: &processSyncKBService{kb: &types.KnowledgeBase{ID: f.ds.KnowledgeBaseID, TenantID: f.ds.TenantID}},
+ }
+ ds := *f.ds
+ ds.ID = "invalid-schedule"
+ ds.Type = deletedItemConnectorType
+ ds.SyncSchedule = "not-a-cron"
+ ds.Config = []byte(`{"type":"test"}`)
+ _, err := svc.CreateDataSource(context.Background(), &ds)
+ require.ErrorContains(t, err, "invalid cron expression")
+ _, err = svc.GetDataSource(context.Background(), ds.ID)
+ require.Error(t, err)
+ require.Equal(t, 1, f.scheduler.EntryCount())
+}
+
+func TestDataSourceScheduleLifecycle(t *testing.T) {
+ for _, schedule := range []string{"", "0 0 */6 * * *", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {
+ t.Run(schedule, func(t *testing.T) {
+ f := newSQLiteDataSourceDeleteFixture(t)
+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
+ updated := *f.ds
+ updated.SyncSchedule = schedule
+ _, err := svc.UpdateDataSource(context.Background(), &updated)
+ require.NoError(t, err)
+ wantEntries := 1
+ if schedule == "" {
+ wantEntries = 0
+ }
+ require.Equal(t, wantEntries, f.scheduler.EntryCount())
+ require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))
+ require.Zero(t, f.scheduler.EntryCount())
+ // Repeated resume must replace, rather than duplicate, the job.
+ for range 2 {
+ require.NoError(t, svc.ResumeDataSource(context.Background(), f.ds.ID))
+ require.Equal(t, wantEntries, f.scheduler.EntryCount())
+ }
+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
+ require.NoError(t, err)
+ require.Equal(t, types.DataSourceStatusActive, saved.Status)
+ require.Equal(t, schedule, saved.SyncSchedule)
+ })
+ }
+}
+
+func TestDataSourceScheduleWriteFailurePreservesState(t *testing.T) {
+ for _, operation := range []string{"update", "resume"} {
+ t.Run(operation, func(t *testing.T) {
+ f := newSQLiteDataSourceDeleteFixture(t)
+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
+ if operation == "resume" {
+ require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))
+ }
+ before, err := svc.GetDataSource(context.Background(), f.ds.ID)
+ require.NoError(t, err)
+ entries := f.scheduler.EntryCount()
+ require.NoError(t, f.db.Exec(`CREATE TRIGGER fail_schedule_write BEFORE UPDATE ON data_sources
+ BEGIN SELECT RAISE(FAIL, 'forced schedule write failure'); END;`).Error)
+ if operation == "resume" {
+ err = svc.ResumeDataSource(context.Background(), f.ds.ID)
+ } else {
+ updated := *before
+ updated.SyncSchedule = "0 30 * * * *"
+ _, err = svc.UpdateDataSource(context.Background(), &updated)
+ }
+ require.ErrorContains(t, err, "forced schedule write failure")
+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
+ require.NoError(t, err)
+ require.Equal(t, before.Status, saved.Status)
+ require.Equal(t, before.SyncSchedule, saved.SyncSchedule)
+ require.Equal(t, entries, f.scheduler.EntryCount())
+ })
+ }
+}
+
+func TestDataSourceScheduleCanceledUpdatePreservesState(t *testing.T) {
+ f := newSQLiteDataSourceDeleteFixture(t)
+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
+ updated := *f.ds
+ updated.SyncSchedule = "0 30 * * * *"
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, err := svc.UpdateDataSource(ctx, &updated)
+ require.ErrorIs(t, err, context.Canceled)
+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
+ require.NoError(t, err)
+ require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)
+ require.Equal(t, 1, f.scheduler.EntryCount())
+}
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index 5886da8fb..fe106dfe1 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -69,6 +69,9 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Data
if ds == nil {
return nil, datasource.ErrDataSourceInvalid
}
+ if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
+ return nil, err
+ }
// Validate knowledge base exists
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID)
@@ -169,6 +172,9 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
if ds.TenantID != existing.TenantID {
return nil, datasource.ErrDataSourceInvalid
}
+ if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
+ return nil, err
+ }
// Credentials NEVER flow through this endpoint — they live behind the
// /credentials subresource. Force-preserve the stored credentials map
@@ -549,6 +555,9 @@ func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) err
if err != nil {
return err
}
+ if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
+ return err
+ }
ds.Status = types.DataSourceStatusActive
if err := s.dsRepo.Update(ctx, ds); err != nil {
diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.go
index 196327fe2..0419d1caa 100644
--- a/internal/datasource/scheduler.go
+++ b/internal/datasource/scheduler.go
@@ -15,6 +15,22 @@ import (
"github.com/robfig/cron/v3"
)
+var syncScheduleParser = cron.NewParser(
+ cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
+)
+
+// ValidateSyncSchedule accepts the same syntax as the scheduler. An empty
+// schedule disables automatic sync and remains valid for manual-only sources.
+func ValidateSyncSchedule(schedule string) error {
+ if schedule == "" {
+ return nil
+ }
+ if _, err := syncScheduleParser.Parse(schedule); err != nil {
+ return fmt.Errorf("invalid cron expression %q: %w", schedule, err)
+ }
+ return nil
+}
+
// Scheduler manages cron-based periodic sync for data sources.
//
// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires
@@ -41,7 +57,7 @@ func NewScheduler(
taskEnqueuer interfaces.TaskEnqueuer,
) *Scheduler {
return &Scheduler{
- cron: cron.New(cron.WithSeconds(), cron.WithChain(
+ cron: cron.New(cron.WithParser(syncScheduleParser), cron.WithChain(
cron.Recover(cron.DefaultLogger),
)),
dsRepo: dsRepo,
@@ -82,6 +98,13 @@ func (s *Scheduler) Stop() {
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error {
+ // Reject an invalid replacement before removing the working entry. Inactive
+ // sources must still be removable even if their stored schedule is invalid.
+ if ds.Status == types.DataSourceStatusActive {
+ if err := ValidateSyncSchedule(ds.SyncSchedule); err != nil {
+ return err
+ }
+ }
s.mu.Lock()
defer s.mu.Unlock()
diff --git a/internal/datasource/scheduler_test.go b/internal/datasource/scheduler_test.go
index dbd35b46b..eafa95c8d 100644
--- a/internal/datasource/scheduler_test.go
+++ b/internal/datasource/scheduler_test.go
@@ -345,6 +345,92 @@ func TestScheduler_InvalidCron(t *testing.T) {
}
}
+func TestScheduler_InvalidReplacementPreservesSchedule(t *testing.T) {
+ repo := newFakeDataSourceRepo()
+ enqueuer := &fakeTaskEnqueuer{}
+ s := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
+ ds := &types.DataSource{ID: "replace", Status: types.DataSourceStatusActive, SyncSchedule: "* * * * * *"}
+ if err := repo.Create(context.Background(), ds); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.AddOrUpdate(ds); err != nil {
+ t.Fatal(err)
+ }
+ invalid := *ds
+ invalid.SyncSchedule = "not-a-cron"
+ if err := s.AddOrUpdate(&invalid); err == nil {
+ t.Fatal("expected invalid schedule to be rejected")
+ }
+ if got := s.EntryCount(); got != 1 {
+ t.Fatalf("invalid replacement removed existing schedule: entries=%d", got)
+ }
+ s.cron.Start()
+ defer s.Stop()
+ deadline := time.Now().Add(3 * time.Second)
+ for enqueuer.count.Load() == 0 && time.Now().Before(deadline) {
+ time.Sleep(10 * time.Millisecond)
+ }
+ if enqueuer.count.Load() == 0 {
+ t.Fatal("preserved schedule did not enqueue a task")
+ }
+}
+
+func TestValidateSyncSchedule(t *testing.T) {
+ for _, schedule := range []string{"", "0 0 */6 * * *", "@hourly", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {
+ t.Run(schedule, func(t *testing.T) {
+ if err := ValidateSyncSchedule(schedule); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+ for _, schedule := range []string{"not-a-cron", "0 0 * * *", "60 * * * * *", "CRON_TZ=Invalid/Zone 0 0 * * * *"} {
+ t.Run(schedule, func(t *testing.T) {
+ if err := ValidateSyncSchedule(schedule); err == nil {
+ t.Fatal("expected invalid schedule to be rejected")
+ }
+ })
+ }
+}
+
+func TestScheduler_ConcurrentReplacements(t *testing.T) {
+ s := NewScheduler(nil, nil, nil)
+ var wg sync.WaitGroup
+ for i := range 20 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ schedule := "0 0 * * * *"
+ if i%2 == 0 {
+ schedule = "invalid"
+ }
+ ds := &types.DataSource{ID: "concurrent", Status: types.DataSourceStatusActive, SyncSchedule: schedule}
+ if err := s.AddOrUpdate(ds); (err != nil) != (i%2 == 0) {
+ t.Errorf("unexpected result for schedule %q: %v", schedule, err)
+ }
+ }()
+ }
+ wg.Wait()
+ if got := s.EntryCount(); got != 1 {
+ t.Fatalf("concurrent replacements left %d entries, want 1", got)
+ }
+}
+
+func TestScheduler_PauseRemovesEntryWithInvalidStoredSchedule(t *testing.T) {
+ s := NewScheduler(nil, nil, nil)
+ ds := &types.DataSource{ID: "pause", Status: types.DataSourceStatusActive, SyncSchedule: "0 0 * * * *"}
+ if err := s.AddOrUpdate(ds); err != nil {
+ t.Fatal(err)
+ }
+ ds.Status = types.DataSourceStatusPaused
+ ds.SyncSchedule = "invalid"
+ if err := s.AddOrUpdate(ds); err != nil {
+ t.Fatal(err)
+ }
+ if got := s.EntryCount(); got != 0 {
+ t.Fatalf("pause left %d entries", got)
+ }
+}
+
func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) {
repo := newFakeDataSourceRepo()
// Create a data source that is paused
diff --git a/internal/handler/datasource_schedule_test.go b/internal/handler/datasource_schedule_test.go
new file mode 100644
index 000000000..f03fcb230
--- /dev/null
+++ b/internal/handler/datasource_schedule_test.go
@@ -0,0 +1,80 @@
+package handler
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/application/repository"
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+)
+
+func TestDataSourceScheduleAPIRejectsInvalidCron(t *testing.T) {
+ for _, operation := range []string{"create", "update", "resume"} {
+ t.Run(operation, func(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "schedule.db")), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&types.DataSource{}))
+ repo := repository.NewDataSourceRepository(db)
+ scheduler := datasource.NewScheduler(repo, nil, nil)
+ kbSvc := &stubKBServiceForDS{getByID: func(context.Context, string) (*types.KnowledgeBase, error) {
+ return &types.KnowledgeBase{ID: "kb-schedule", TenantID: 1}, nil
+ }}
+ svc := service.NewDataSourceService(repo, nil, nil, kbSvc, nil, nil, scheduler, nil, nil, nil)
+ ds := &types.DataSource{
+ ID: "ds-schedule", KnowledgeBaseID: "kb-schedule", TenantID: 1,
+ Type: types.ConnectorTypeRSS, Status: types.DataSourceStatusActive,
+ SyncSchedule: "0 0 * * * *",
+ }
+ if operation == "resume" {
+ ds.Status = types.DataSourceStatusPaused
+ ds.SyncSchedule = "invalid"
+ }
+ if operation != "create" {
+ require.NoError(t, repo.Create(context.Background(), ds))
+ require.NoError(t, scheduler.AddOrUpdate(ds))
+ }
+ entries := scheduler.EntryCount()
+ h := NewDataSourceHandler(svc, kbSvc)
+ router := newDataSourceTestRouter(h)
+ router.POST("/datasource", h.CreateDataSource)
+ router.PUT("/datasource/:id", h.UpdateDataSource)
+ router.POST("/datasource/:id/resume", h.ResumeDataSource)
+ body := *ds
+ body.SyncSchedule = "invalid"
+ payload, err := json.Marshal(body)
+ require.NoError(t, err)
+ method, url := http.MethodPost, "/datasource"
+ switch operation {
+ case "update":
+ method, url = http.MethodPut, "/datasource/"+ds.ID
+ case "resume":
+ url = "/datasource/" + ds.ID + "/resume"
+ }
+ req := withDSCtx(httptest.NewRequest(method, url, bytes.NewReader(payload)), 1)
+ req.Header.Set("Content-Type", "application/json")
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, req)
+ require.Equal(t, http.StatusBadRequest, response.Code, response.Body.String())
+ require.Contains(t, response.Body.String(), "invalid cron expression")
+ require.Equal(t, entries, scheduler.EntryCount())
+ saved, err := svc.GetDataSource(context.Background(), ds.ID)
+ if operation == "create" {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, ds.Status, saved.Status)
+ require.Equal(t, ds.SyncSchedule, saved.SyncSchedule)
+ }
+ })
+ }
+}
diff --git a/website-docs/03-features/10-datasource.md b/website-docs/03-features/10-datasource.md
index 9439ca201..8441c6df9 100644
--- a/website-docs/03-features/10-datasource.md
+++ b/website-docs/03-features/10-datasource.md
@@ -20,6 +20,8 @@
凭据更新需要提交完整配置,系统会在线验证。暂停数据源可停止后续计划同步;恢复后重新注册调度。
+`sync_schedule` 使用包含秒的六段 cron,例如 `0 0 */6 * * *` 表示每六小时触发;空字符串表示仅手动同步。创建、编辑及恢复数据源时会校验表达式,无效值返回 HTTP 400,不改变已保存配置或已有定时任务。历史数据源若保存了无效表达式,需先修改为合法表达式或清空,再恢复同步。
+
## 选择连接器
| 连接器 | 类型标识 | 同步对象 |
diff --git a/website-docs/04-api/02-api-infra.md b/website-docs/04-api/02-api-infra.md
index 847c96872..9c61c6996 100644
--- a/website-docs/04-api/02-api-infra.md
+++ b/website-docs/04-api/02-api-infra.md
@@ -374,11 +374,17 @@ curl $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN"
用途:更新(`id/tenant_id/knowledge_base_id` 锁定为原值)。权限:Admin+。请求体同创建。
+PUT 整体替换数据源配置,不支持部分更新:请先 GET 详情,修改后回传完整对象(`name`、`type`、`config`、`sync_schedule`、`sync_mode`、`conflict_strategy`、`sync_deletions`、`sync_log_retention_days`、`status` 等)。`sync_schedule` 和 `sync_deletions` 总是按请求体写入:省略 `sync_schedule` 会清空为空字符串(仅手动同步,已有定时任务随之移除),省略 `sync_deletions` 即为 false;其他字段省略时是否保留原值不作保证。凭证不经本接口修改,请求体中的 `config.credentials` 会被忽略,保留已存凭证。
+
响应:200 `DataSourceResponse`
```bash
curl -X PUT $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN" \
- -H 'Content-Type: application/json' -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1","config":{}}'
+ -H 'Content-Type: application/json' \
+ -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1",
+ "config":{"type":"notion","resource_ids":["page-1"]},
+ "sync_schedule":"0 0 */6 * * *","sync_mode":"incremental","conflict_strategy":"overwrite",
+ "sync_deletions":true,"sync_log_retention_days":30,"status":"active"}'
```
### DELETE /api/v1/datasource/:id