Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): validate cron schedules and persist manual-only mode (#3653)
Perfecto00e6366fd27a
diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.goindex c3b2f58e4..8f2cabbe0 100644--- a/internal/application/repository/datasource_repo.go+++ b/internal/application/repository/datasource_repo.go@@ -90,10 +90,13 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource)return err}// GORM Updates(struct) deliberately skips zero values, which would make- // a user-selected sync_deletions=false impossible to persist.+ // a user-selected sync_deletions=false or manual-only schedule impossible to persist.return tx.Model(&types.DataSource{}).Where("id = ?", ds.ID).- UpdateColumn("sync_deletions", ds.SyncDeletions).Error+ UpdateColumns(map[string]interface{}{+ "sync_deletions": ds.SyncDeletions,+ "sync_schedule": ds.SyncSchedule,+ }).Error})}diff --git a/internal/application/service/datasource_schedule_test.go b/internal/application/service/datasource_schedule_test.gonew file mode 100644index 000000000..0320bd3f3--- /dev/null+++ b/internal/application/service/datasource_schedule_test.go@@ -0,0 +1,133 @@+package service++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/datasource"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++func TestDataSourceResumeRejectsInvalidSchedule(t *testing.T) {+ f := newSQLiteDataSourceDeleteFixture(t)+ f.scheduler.Remove(f.ds.ID)+ f.ds.Status = types.DataSourceStatusPaused+ f.ds.SyncSchedule = "not-a-cron"+ require.NoError(t, f.dsRepo.Update(context.Background(), f.ds))+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}+ err := svc.ResumeDataSource(context.Background(), f.ds.ID)+ saved, readErr := svc.GetDataSource(context.Background(), f.ds.ID)+ require.NoError(t, readErr)+ t.Logf("resume error=%v; status=%s; entries=%d", err, saved.Status, f.scheduler.EntryCount())+ require.ErrorContains(t, err, "invalid cron expression")+ require.Equal(t, types.DataSourceStatusPaused, saved.Status)+ require.Zero(t, f.scheduler.EntryCount())+}++func TestDataSourceUpdateRejectsInvalidSchedule(t *testing.T) {+ f := newSQLiteDataSourceDeleteFixture(t)+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}+ updated := *f.ds+ updated.SyncSchedule = "not-a-cron"+ _, err := svc.UpdateDataSource(context.Background(), &updated)+ require.ErrorContains(t, err, "invalid cron expression")+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)+ require.NoError(t, err)+ require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)+ require.Equal(t, 1, f.scheduler.EntryCount())+}++func TestDataSourceCreateRejectsInvalidSchedule(t *testing.T) {+ f := newSQLiteDataSourceDeleteFixture(t)+ registry := datasource.NewConnectorRegistry()+ require.NoError(t, registry.Register(deletedItemConnector{}))+ svc := &DataSourceService{+ dsRepo: f.dsRepo, scheduler: f.scheduler, connectorRegistry: registry,+ kbService: &processSyncKBService{kb: &types.KnowledgeBase{ID: f.ds.KnowledgeBaseID, TenantID: f.ds.TenantID}},+ }+ ds := *f.ds+ ds.ID = "invalid-schedule"+ ds.Type = deletedItemConnectorType+ ds.SyncSchedule = "not-a-cron"+ ds.Config = []byte(`{"type":"test"}`)+ _, err := svc.CreateDataSource(context.Background(), &ds)+ require.ErrorContains(t, err, "invalid cron expression")+ _, err = svc.GetDataSource(context.Background(), ds.ID)+ require.Error(t, err)+ require.Equal(t, 1, f.scheduler.EntryCount())+}++func TestDataSourceScheduleLifecycle(t *testing.T) {+ for _, schedule := range []string{"", "0 0 */6 * * *", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {+ t.Run(schedule, func(t *testing.T) {+ f := newSQLiteDataSourceDeleteFixture(t)+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}+ updated := *f.ds+ updated.SyncSchedule = schedule+ _, err := svc.UpdateDataSource(context.Background(), &updated)+ require.NoError(t, err)+ wantEntries := 1+ if schedule == "" {+ wantEntries = 0+ }+ require.Equal(t, wantEntries, f.scheduler.EntryCount())+ require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))+ require.Zero(t, f.scheduler.EntryCount())+ // Repeated resume must replace, rather than duplicate, the job.+ for range 2 {+ require.NoError(t, svc.ResumeDataSource(context.Background(), f.ds.ID))+ require.Equal(t, wantEntries, f.scheduler.EntryCount())+ }+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)+ require.NoError(t, err)+ require.Equal(t, types.DataSourceStatusActive, saved.Status)+ require.Equal(t, schedule, saved.SyncSchedule)+ })+ }+}++func TestDataSourceScheduleWriteFailurePreservesState(t *testing.T) {+ for _, operation := range []string{"update", "resume"} {+ t.Run(operation, func(t *testing.T) {+ f := newSQLiteDataSourceDeleteFixture(t)+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}+ if operation == "resume" {+ require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))+ }+ before, err := svc.GetDataSource(context.Background(), f.ds.ID)+ require.NoError(t, err)+ entries := f.scheduler.EntryCount()+ require.NoError(t, f.db.Exec(`CREATE TRIGGER fail_schedule_write BEFORE UPDATE ON data_sources+ BEGIN SELECT RAISE(FAIL, 'forced schedule write failure'); END;`).Error)+ if operation == "resume" {+ err = svc.ResumeDataSource(context.Background(), f.ds.ID)+ } else {+ updated := *before+ updated.SyncSchedule = "0 30 * * * *"+ _, err = svc.UpdateDataSource(context.Background(), &updated)+ }+ require.ErrorContains(t, err, "forced schedule write failure")+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)+ require.NoError(t, err)+ require.Equal(t, before.Status, saved.Status)+ require.Equal(t, before.SyncSchedule, saved.SyncSchedule)+ require.Equal(t, entries, f.scheduler.EntryCount())+ })+ }+}++func TestDataSourceScheduleCanceledUpdatePreservesState(t *testing.T) {+ f := newSQLiteDataSourceDeleteFixture(t)+ svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}+ updated := *f.ds+ updated.SyncSchedule = "0 30 * * * *"+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ _, err := svc.UpdateDataSource(ctx, &updated)+ require.ErrorIs(t, err, context.Canceled)+ saved, err := svc.GetDataSource(context.Background(), f.ds.ID)+ require.NoError(t, err)+ require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)+ require.Equal(t, 1, f.scheduler.EntryCount())+}diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.goindex 5886da8fb..fe106dfe1 100644--- a/internal/application/service/datasource_service.go+++ b/internal/application/service/datasource_service.go@@ -69,6 +69,9 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Dataif ds == nil {return nil, datasource.ErrDataSourceInvalid}+ if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {+ return nil, err+ }// Validate knowledge base existskb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID)@@ -169,6 +172,9 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Dataif ds.TenantID != existing.TenantID {return nil, datasource.ErrDataSourceInvalid}+ if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {+ return nil, err+ }// Credentials NEVER flow through this endpoint — they live behind the// /credentials subresource. Force-preserve the stored credentials map@@ -549,6 +555,9 @@ func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) errif err != nil {return err}+ if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {+ return err+ }ds.Status = types.DataSourceStatusActiveif err := s.dsRepo.Update(ctx, ds); err != nil {diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.goindex 196327fe2..0419d1caa 100644--- a/internal/datasource/scheduler.go+++ b/internal/datasource/scheduler.go@@ -15,6 +15,22 @@ import ("github.com/robfig/cron/v3")+var syncScheduleParser = cron.NewParser(+ cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,+)++// ValidateSyncSchedule accepts the same syntax as the scheduler. An empty+// schedule disables automatic sync and remains valid for manual-only sources.+func ValidateSyncSchedule(schedule string) error {+ if schedule == "" {+ return nil+ }+ if _, err := syncScheduleParser.Parse(schedule); err != nil {+ return fmt.Errorf("invalid cron expression %q: %w", schedule, err)+ }+ return nil+}+// Scheduler manages cron-based periodic sync for data sources.//// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires@@ -41,7 +57,7 @@ func NewScheduler(taskEnqueuer interfaces.TaskEnqueuer,) *Scheduler {return &Scheduler{- cron: cron.New(cron.WithSeconds(), cron.WithChain(+ cron: cron.New(cron.WithParser(syncScheduleParser), cron.WithChain(cron.Recover(cron.DefaultLogger),)),dsRepo: dsRepo,@@ -82,6 +98,13 @@ func (s *Scheduler) Stop() {// AddOrUpdate registers (or re-registers) a cron entry for the given data source.func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error {+ // Reject an invalid replacement before removing the working entry. Inactive+ // sources must still be removable even if their stored schedule is invalid.+ if ds.Status == types.DataSourceStatusActive {+ if err := ValidateSyncSchedule(ds.SyncSchedule); err != nil {+ return err+ }+ }s.mu.Lock()defer s.mu.Unlock()diff --git a/internal/datasource/scheduler_test.go b/internal/datasource/scheduler_test.goindex dbd35b46b..eafa95c8d 100644--- a/internal/datasource/scheduler_test.go+++ b/internal/datasource/scheduler_test.go@@ -345,6 +345,92 @@ func TestScheduler_InvalidCron(t *testing.T) {}}+func TestScheduler_InvalidReplacementPreservesSchedule(t *testing.T) {+ repo := newFakeDataSourceRepo()+ enqueuer := &fakeTaskEnqueuer{}+ s := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)+ ds := &types.DataSource{ID: "replace", Status: types.DataSourceStatusActive, SyncSchedule: "* * * * * *"}+ if err := repo.Create(context.Background(), ds); err != nil {+ t.Fatal(err)+ }+ if err := s.AddOrUpdate(ds); err != nil {+ t.Fatal(err)+ }+ invalid := *ds+ invalid.SyncSchedule = "not-a-cron"+ if err := s.AddOrUpdate(&invalid); err == nil {+ t.Fatal("expected invalid schedule to be rejected")+ }+ if got := s.EntryCount(); got != 1 {+ t.Fatalf("invalid replacement removed existing schedule: entries=%d", got)+ }+ s.cron.Start()+ defer s.Stop()+ deadline := time.Now().Add(3 * time.Second)+ for enqueuer.count.Load() == 0 && time.Now().Before(deadline) {+ time.Sleep(10 * time.Millisecond)+ }+ if enqueuer.count.Load() == 0 {+ t.Fatal("preserved schedule did not enqueue a task")+ }+}++func TestValidateSyncSchedule(t *testing.T) {+ for _, schedule := range []string{"", "0 0 */6 * * *", "@hourly", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {+ t.Run(schedule, func(t *testing.T) {+ if err := ValidateSyncSchedule(schedule); err != nil {+ t.Fatal(err)+ }+ })+ }+ for _, schedule := range []string{"not-a-cron", "0 0 * * *", "60 * * * * *", "CRON_TZ=Invalid/Zone 0 0 * * * *"} {+ t.Run(schedule, func(t *testing.T) {+ if err := ValidateSyncSchedule(schedule); err == nil {+ t.Fatal("expected invalid schedule to be rejected")+ }+ })+ }+}++func TestScheduler_ConcurrentReplacements(t *testing.T) {+ s := NewScheduler(nil, nil, nil)+ var wg sync.WaitGroup+ for i := range 20 {+ wg.Add(1)+ go func() {+ defer wg.Done()+ schedule := "0 0 * * * *"+ if i%2 == 0 {+ schedule = "invalid"+ }+ ds := &types.DataSource{ID: "concurrent", Status: types.DataSourceStatusActive, SyncSchedule: schedule}+ if err := s.AddOrUpdate(ds); (err != nil) != (i%2 == 0) {+ t.Errorf("unexpected result for schedule %q: %v", schedule, err)+ }+ }()+ }+ wg.Wait()+ if got := s.EntryCount(); got != 1 {+ t.Fatalf("concurrent replacements left %d entries, want 1", got)+ }+}++func TestScheduler_PauseRemovesEntryWithInvalidStoredSchedule(t *testing.T) {+ s := NewScheduler(nil, nil, nil)+ ds := &types.DataSource{ID: "pause", Status: types.DataSourceStatusActive, SyncSchedule: "0 0 * * * *"}+ if err := s.AddOrUpdate(ds); err != nil {+ t.Fatal(err)+ }+ ds.Status = types.DataSourceStatusPaused+ ds.SyncSchedule = "invalid"+ if err := s.AddOrUpdate(ds); err != nil {+ t.Fatal(err)+ }+ if got := s.EntryCount(); got != 0 {+ t.Fatalf("pause left %d entries", got)+ }+}+func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) {repo := newFakeDataSourceRepo()// Create a data source that is pauseddiff --git a/internal/handler/datasource_schedule_test.go b/internal/handler/datasource_schedule_test.gonew file mode 100644index 000000000..f03fcb230--- /dev/null+++ b/internal/handler/datasource_schedule_test.go@@ -0,0 +1,80 @@+package handler++import (+ "bytes"+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "path/filepath"+ "testing"++ "github.com/Tencent/WeKnora/internal/application/repository"+ "github.com/Tencent/WeKnora/internal/application/service"+ "github.com/Tencent/WeKnora/internal/datasource"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+)++func TestDataSourceScheduleAPIRejectsInvalidCron(t *testing.T) {+ for _, operation := range []string{"create", "update", "resume"} {+ t.Run(operation, func(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "schedule.db")), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&types.DataSource{}))+ repo := repository.NewDataSourceRepository(db)+ scheduler := datasource.NewScheduler(repo, nil, nil)+ kbSvc := &stubKBServiceForDS{getByID: func(context.Context, string) (*types.KnowledgeBase, error) {+ return &types.KnowledgeBase{ID: "kb-schedule", TenantID: 1}, nil+ }}+ svc := service.NewDataSourceService(repo, nil, nil, kbSvc, nil, nil, scheduler, nil, nil, nil)+ ds := &types.DataSource{+ ID: "ds-schedule", KnowledgeBaseID: "kb-schedule", TenantID: 1,+ Type: types.ConnectorTypeRSS, Status: types.DataSourceStatusActive,+ SyncSchedule: "0 0 * * * *",+ }+ if operation == "resume" {+ ds.Status = types.DataSourceStatusPaused+ ds.SyncSchedule = "invalid"+ }+ if operation != "create" {+ require.NoError(t, repo.Create(context.Background(), ds))+ require.NoError(t, scheduler.AddOrUpdate(ds))+ }+ entries := scheduler.EntryCount()+ h := NewDataSourceHandler(svc, kbSvc)+ router := newDataSourceTestRouter(h)+ router.POST("/datasource", h.CreateDataSource)+ router.PUT("/datasource/:id", h.UpdateDataSource)+ router.POST("/datasource/:id/resume", h.ResumeDataSource)+ body := *ds+ body.SyncSchedule = "invalid"+ payload, err := json.Marshal(body)+ require.NoError(t, err)+ method, url := http.MethodPost, "/datasource"+ switch operation {+ case "update":+ method, url = http.MethodPut, "/datasource/"+ds.ID+ case "resume":+ url = "/datasource/" + ds.ID + "/resume"+ }+ req := withDSCtx(httptest.NewRequest(method, url, bytes.NewReader(payload)), 1)+ req.Header.Set("Content-Type", "application/json")+ response := httptest.NewRecorder()+ router.ServeHTTP(response, req)+ require.Equal(t, http.StatusBadRequest, response.Code, response.Body.String())+ require.Contains(t, response.Body.String(), "invalid cron expression")+ require.Equal(t, entries, scheduler.EntryCount())+ saved, err := svc.GetDataSource(context.Background(), ds.ID)+ if operation == "create" {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ require.Equal(t, ds.Status, saved.Status)+ require.Equal(t, ds.SyncSchedule, saved.SyncSchedule)+ }+ })+ }+}diff --git a/website-docs/03-features/10-datasource.md b/website-docs/03-features/10-datasource.mdindex 9439ca201..8441c6df9 100644--- a/website-docs/03-features/10-datasource.md+++ b/website-docs/03-features/10-datasource.md@@ -20,6 +20,8 @@凭据更新需要提交完整配置,系统会在线验证。暂停数据源可停止后续计划同步;恢复后重新注册调度。+`sync_schedule` 使用包含秒的六段 cron,例如 `0 0 */6 * * *` 表示每六小时触发;空字符串表示仅手动同步。创建、编辑及恢复数据源时会校验表达式,无效值返回 HTTP 400,不改变已保存配置或已有定时任务。历史数据源若保存了无效表达式,需先修改为合法表达式或清空,再恢复同步。+## 选择连接器| 连接器 | 类型标识 | 同步对象 |diff --git a/website-docs/04-api/02-api-infra.md b/website-docs/04-api/02-api-infra.mdindex 847c96872..9c61c6996 100644--- a/website-docs/04-api/02-api-infra.md+++ b/website-docs/04-api/02-api-infra.md@@ -374,11 +374,17 @@ curl $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN"用途:更新(`id/tenant_id/knowledge_base_id` 锁定为原值)。权限:Admin+。请求体同创建。+PUT 整体替换数据源配置,不支持部分更新:请先 GET 详情,修改后回传完整对象(`name`、`type`、`config`、`sync_schedule`、`sync_mode`、`conflict_strategy`、`sync_deletions`、`sync_log_retention_days`、`status` 等)。`sync_schedule` 和 `sync_deletions` 总是按请求体写入:省略 `sync_schedule` 会清空为空字符串(仅手动同步,已有定时任务随之移除),省略 `sync_deletions` 即为 false;其他字段省略时是否保留原值不作保证。凭证不经本接口修改,请求体中的 `config.credentials` 会被忽略,保留已存凭证。+响应:200 `DataSourceResponse````bashcurl -X PUT $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN" \- -H 'Content-Type: application/json' -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1","config":{}}'+ -H 'Content-Type: application/json' \+ -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1",+ "config":{"type":"notion","resource_ids":["page-1"]},+ "sync_schedule":"0 0 */6 * * *","sync_mode":"incremental","conflict_strategy":"overwrite",+ "sync_deletions":true,"sync_log_retention_days":30,"status":"active"}'```### DELETE /api/v1/datasource/:id