Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): enforce sync log retention policies (#4532)
Luciene18fb23ed12e
diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.goindex 4fbc63498..09de8298e 100644--- a/internal/application/repository/datasource_repo.go+++ b/internal/application/repository/datasource_repo.go@@ -312,17 +312,60 @@ func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID}).Error}-// CleanupOldLogs deletes sync logs older than the retention period-func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays int) error {- if retentionDays <= 0 {- retentionDays = 30- }- // Delete logs older than the retention period- if err := r.db.WithContext(ctx).- Where("started_at < NOW() - INTERVAL ? DAY", retentionDays).- Delete(&types.SyncLog{}).Error; err != nil {+// CleanupOldLogs applies each source's retention policy, including soft-deleted+// sources. Missing sources and non-positive policies use the default (normally+// 30 days). Failed logs of live sources may still be retried by Asynq.+func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, defaultRetentionDays int) error {+ if defaultRetentionDays <= 0 {+ defaultRetentionDays = 30+ }+ var periods []int+ if err := r.db.WithContext(ctx).Unscoped().Model(&types.DataSource{}).+ Where("sync_log_retention_days > 0 AND sync_log_retention_days <> ?", defaultRetentionDays).+ Distinct().Pluck("sync_log_retention_days", &periods).Error; err != nil {return err}+ periods = append(periods, defaultRetentionDays)+ now := time.Now().UTC()+ sourceByID := "data_sources.id = sync_logs.data_source_id"+ sourceMatch := sourceByID + " AND data_sources.tenant_id = sync_logs.tenant_id"+ liveSource := r.db.WithContext(ctx).Model(&types.DataSource{}).Select("1").Where(sourceByID)+ terminal := r.db.WithContext(ctx).+ Where("status IN ?", []string{+ types.SyncLogStatusSuccess, types.SyncLogStatusPartial, types.SyncLogStatusCanceled,+ }).+ Or("status = ? AND NOT EXISTS (?)", types.SyncLogStatusFailed, liveSource)+ for _, days := range periods {+ // A very large user policy must not overflow a time.Duration and turn+ // into a recent cutoff. No valid log can predate year 1.+ if days > now.Year()*366 {+ continue+ }+ cutoff := now.AddDate(0, 0, -days)+ if cutoff.Year() < 1 {+ continue+ }+ sources := r.db.WithContext(ctx).Unscoped().Model(&types.DataSource{}).+ Select("1")+ policy := "EXISTS (?)"+ if days == defaultRetentionDays {+ // Also retire orphan logs after the default horizon instead of+ // keeping them forever once their source has been removed.+ policy = "NOT EXISTS (?)"+ sources = sources.Where(sourceByID).+ Where("sync_log_retention_days > 0 AND sync_log_retention_days <> ?", days)+ } else {+ sources = sources.Where(sourceMatch).+ Where("sync_log_retention_days = ?", days)+ }+ if err := r.db.WithContext(ctx).+ Where(terminal).+ Where("COALESCE(finished_at, started_at) < ?", cutoff).+ Where(policy, sources).+ Delete(&types.SyncLog{}).Error; err != nil {+ return err+ }+ }return nil}diff --git a/internal/application/repository/datasource_retention_test.go b/internal/application/repository/datasource_retention_test.gonew file mode 100644index 000000000..1b5ce2404--- /dev/null+++ b/internal/application/repository/datasource_retention_test.go@@ -0,0 +1,120 @@+package repository++import (+ "context"+ "os"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/postgres"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+)++func TestSyncLogRetentionPerSource(t *testing.T) {+ for _, backend := range []string{"sqlite", "postgres"} {+ t.Run(backend, func(t *testing.T) {+ dialect := sqlite.Open(":memory:")+ if backend == "postgres" {+ dsn := os.Getenv("WEKNORA_REPOSITORY_TEST_POSTGRES_DSN")+ if dsn == "" {+ t.Skip("set WEKNORA_REPOSITORY_TEST_POSTGRES_DSN for PostgreSQL sync log retention tests")+ }+ dialect = postgres.Open(dsn)+ }+ db, err := gorm.Open(dialect, &gorm.Config{})+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ db = db.Begin()+ require.NoError(t, db.Error)+ t.Cleanup(func() { require.NoError(t, db.Rollback().Error) })+ // Temporary tables exercise both dialects without modifying any+ // existing deployment data when an integration-test DSN is supplied.+ require.NoError(t, db.Exec(`CREATE TEMP TABLE data_sources (+ id TEXT PRIMARY KEY, tenant_id BIGINT, sync_log_retention_days INT, deleted_at TIMESTAMP+ )`).Error)+ require.NoError(t, db.Exec(`CREATE TEMP TABLE sync_logs (+ id TEXT PRIMARY KEY, data_source_id TEXT, tenant_id BIGINT, status TEXT,+ started_at TIMESTAMP, finished_at TIMESTAMP+ )`).Error)+ now := time.Now().UTC()+ for _, source := range []struct {+ id string+ tenant uint64+ days int+ deleted bool+ }{+ {"short", 1, 1, false},+ {"long", 2, 30, false},+ {"unrelated", 3, 90, false},+ {"unset", 1, 0, false},+ {"negative", 1, -1, false},+ {"deleted-long", 2, 90, true},+ {"deleted-short", 2, 1, true},+ {"large", 1, 1 << 30, false},+ } {+ var deletedAt *time.Time+ if source.deleted {+ deletedAt = &now+ }+ require.NoError(t, db.Exec("INSERT INTO data_sources VALUES (?, ?, ?, ?)",+ source.id, source.tenant, source.days, deletedAt).Error)+ }+ var want []string+ for _, log := range []struct {+ id string+ source string+ tenant uint64+ status string+ ageDays int+ finishedAge int+ keep bool+ }{+ {"short-expired", "short", 1, types.SyncLogStatusSuccess, 2, 2, false},+ {"partial-expired", "short", 1, types.SyncLogStatusPartial, 2, 2, false},+ {"canceled-expired", "short", 1, types.SyncLogStatusCanceled, 2, 2, false},+ {"recent", "short", 1, types.SyncLogStatusSuccess, 0, 0, true},+ {"recently-finished", "short", 1, types.SyncLogStatusSuccess, 60, 0, true},+ {"legacy-expired", "short", 1, types.SyncLogStatusSuccess, 2, -1, false},+ {"running", "short", 1, types.SyncLogStatusRunning, 60, -1, true},+ {"pending", "short", 1, "pending", 60, -1, true},+ {"retry", "short", 1, "retry", 60, -1, true},+ {"failed-retryable", "short", 1, types.SyncLogStatusFailed, 60, 60, true},+ {"long-retained", "long", 2, types.SyncLogStatusSuccess, 2, 2, true},+ {"long-expired", "long", 2, types.SyncLogStatusSuccess, 31, 31, false},+ {"unrelated-retained", "unrelated", 3, types.SyncLogStatusSuccess, 60, 60, true},+ {"unset-expired", "unset", 1, types.SyncLogStatusSuccess, 31, 31, false},+ {"negative-retained", "negative", 1, types.SyncLogStatusSuccess, 2, 2, true},+ {"negative-expired", "negative", 1, types.SyncLogStatusSuccess, 31, 31, false},+ {"deleted-policy-retained", "deleted-long", 2, types.SyncLogStatusSuccess, 60, 60, true},+ {"deleted-policy-expired", "deleted-short", 2, types.SyncLogStatusFailed, 2, 2, false},+ {"orphan-retained", "missing", 1, types.SyncLogStatusSuccess, 2, 2, true},+ {"orphan-expired", "missing", 1, types.SyncLogStatusFailed, 31, 31, false},+ {"tenant-mismatch-retained", "short", 9, types.SyncLogStatusSuccess, 60, 60, true},+ {"large-policy-retained", "large", 1, types.SyncLogStatusSuccess, 60, 60, true},+ } {+ var finishedAt *time.Time+ if log.finishedAge >= 0 {+ finished := now.AddDate(0, 0, -log.finishedAge)+ finishedAt = &finished+ }+ require.NoError(t, db.Exec("INSERT INTO sync_logs VALUES (?, ?, ?, ?, ?, ?)",+ log.id, log.source, log.tenant, log.status, now.AddDate(0, 0, -log.ageDays), finishedAt).Error)+ if log.keep {+ want = append(want, log.id)+ }+ }+ repo := NewSyncLogRepository(db)+ for _, defaultDays := range []int{30, 0} {+ require.NoError(t, repo.CleanupOldLogs(context.Background(), defaultDays))+ var got []string+ require.NoError(t, db.Table("sync_logs").Pluck("id", &got).Error)+ require.ElementsMatch(t, want, got, "cleanup must be isolated and idempotent")+ }+ })+ }+}diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.goindex fd5954458..c81e28430 100644--- a/internal/datasource/scheduler.go+++ b/internal/datasource/scheduler.go@@ -85,12 +85,27 @@ func (s *Scheduler) Start(ctx context.Context) error {ds.ID, ds.SyncSchedule, err)}}+ // Retention also runs for paused and manual-only sources. Reuse this+ // runner so StopWithin waits for cleanup along with in-flight sync jobs.+ if _, err := s.cron.AddJob("@daily", cron.NewChain(+ cron.SkipIfStillRunning(cron.DefaultLogger),+ ).Then(cron.FuncJob(s.cleanupSyncLogs))); err != nil {+ return fmt.Errorf("register sync log retention: %w", err)+ }s.cron.Start()logger.Infof(ctx, "[Scheduler] started with %d cron entries", len(s.entries))return nil}+func (s *Scheduler) cleanupSyncLogs() {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ if err := s.syncLogRepo.CleanupOldLogs(ctx, 30); err != nil {+ logger.Warnf(ctx, "[Scheduler] sync log retention failed: %v", err)+ }+}+// Stop gracefully stops the cron runner and waits for running jobs to finish.func (s *Scheduler) Stop() {s.StopWithin(0)diff --git a/internal/datasource/scheduler_retention_test.go b/internal/datasource/scheduler_retention_test.gonew file mode 100644index 000000000..e42e5057f--- /dev/null+++ b/internal/datasource/scheduler_retention_test.go@@ -0,0 +1,82 @@+package datasource++import (+ "context"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++type retentionCall struct {+ ctx context.Context+ days int+}++type retentionSyncLogRepo struct {+ *fakeSyncLogRepo+ calls chan retentionCall+ release <-chan struct{}+}++func (r *retentionSyncLogRepo) CleanupOldLogs(ctx context.Context, days int) error {+ r.calls <- retentionCall{ctx, days}+ select {+ case <-r.release:+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+}++func TestSchedulerRunsDailyRetentionAndWaitsForShutdown(t *testing.T) {+ release := make(chan struct{})+ var releaseOnce sync.Once+ unblock := func() { releaseOnce.Do(func() { close(release) }) }+ repo := &retentionSyncLogRepo{newFakeSyncLogRepo(), make(chan retentionCall, 1), release}+ scheduler := NewScheduler(newFakeDataSourceRepo(), repo, &fakeTaskEnqueuer{})+ require.NoError(t, scheduler.Start(context.Background()))+ t.Cleanup(scheduler.Stop)+ t.Cleanup(unblock)++ // Retention must be registered even without a source cron entry, so+ // paused/manual-only/deleted sources are also eventually cleaned.+ require.Zero(t, scheduler.EntryCount())+ entries := scheduler.cron.Entries()+ require.Len(t, entries, 1)+ now := time.Now()+ want := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location())+ require.Equal(t, want, entries[0].Schedule.Next(now))++ // Run the registered production job through cron at a shorter test+ // interval; cron's in-flight wait group must include the retention call.+ scheduler.cron.Remove(entries[0].ID)+ _, err := scheduler.cron.AddJob("@every 1s", entries[0].Job)+ require.NoError(t, err)+ select {+ case call := <-repo.calls:+ require.Equal(t, 30, call.days)+ deadline, ok := call.ctx.Deadline()+ require.True(t, ok, "retention database calls must have a timeout")+ require.InDelta(t, 30, time.Until(deadline).Seconds(), 2)+ case <-time.After(3 * time.Second):+ t.Fatal("retention job did not run")+ }+ stopped := make(chan struct{})+ go func() {+ scheduler.Stop()+ close(stopped)+ }()+ select {+ case <-stopped:+ t.Fatal("scheduler stopped before its retention call finished")+ case <-time.After(20 * time.Millisecond):+ }+ unblock()+ select {+ case <-stopped:+ case <-time.After(3 * time.Second):+ t.Fatal("scheduler failed to stop after retention completed")+ }+}diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.goindex a24a05a06..e39da0171 100644--- a/internal/types/interfaces/datasource.go+++ b/internal/types/interfaces/datasource.go@@ -135,6 +135,8 @@ type SyncLogRepository interface {// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.CancelPendingByDataSource(ctx context.Context, dsID string) error- // CleanupOldLogs deletes sync logs older than the retention period- CleanupOldLogs(ctx context.Context, retentionDays int) error+ // CleanupOldLogs deletes expired terminal logs using each source's policy.+ // Failed logs of live sources are retained because they may be awaiting retry.+ // defaultRetentionDays applies to missing sources and unset policies.+ CleanupOldLogs(ctx context.Context, defaultRetentionDays int) error}diff --git a/website-docs/04-api/02-api-infra.md b/website-docs/04-api/02-api-infra.mdindex 9c61c6996..83956ce02 100644--- a/website-docs/04-api/02-api-infra.md+++ b/website-docs/04-api/02-api-infra.md@@ -342,10 +342,12 @@ curl -X POST $BASE/api/v1/datasource/validate-credentials -H "Authorization: Bea| `sync_mode` | string | 否 | `incremental`(默认)/`full` || `conflict_strategy` | string | 否 | `overwrite`(默认)/`skip` || `sync_deletions` | bool | 否 | 默认 true |-| `sync_log_retention_days` | int | 否 | 默认 30 |+| `sync_log_retention_days` | int | 否 | 本数据源同步日志保留天数,默认 30;非正值按 30 天处理 |响应:201 `DataSourceResponse`(凭证剥离,见 `internal/handler/dto/datasource.go`)。+后台每日凌晨(服务所在时区)清理过期的 `success`、`partial`、`canceled` 同步日志,包括暂停和仅手动同步的数据源。保留期从完成时间计算,旧日志缺少完成时间时使用开始时间。运行中或等待执行的日志不清理;现有数据源的 `failed` 日志也会保留,因为该状态可能仍在等待任务重试。数据源软删除后继续沿用原保留期,并可清理 `failed` 日志;无来源日志按默认 30 天保留。+```bashcurl -X POST $BASE/api/v1/datasource -H "Authorization: Bearer $TOKEN" \-H 'Content-Type: application/json' \