Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): enforce sync log retention policies (#4532)

Luciene18fb23ed12e
diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go
index 4fbc63498..09de8298e 100644
--- a/internal/application/repository/datasource_repo.go
+++ b/internal/application/repository/datasource_repo.go
@@ -312,17 +312,60 @@ func (r *SyncLogRepository) CancelPendingByDataSource(ctx context.Context, dsID
}).Error
}
-// CleanupOldLogs deletes sync logs older than the retention period
-func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays int) error {
- if retentionDays <= 0 {
- retentionDays = 30
- }
- // Delete logs older than the retention period
- if err := r.db.WithContext(ctx).
- Where("started_at < NOW() - INTERVAL ? DAY", retentionDays).
- Delete(&types.SyncLog{}).Error; err != nil {
+// CleanupOldLogs applies each source's retention policy, including soft-deleted
+// sources. Missing sources and non-positive policies use the default (normally
+// 30 days). Failed logs of live sources may still be retried by Asynq.
+func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, defaultRetentionDays int) error {
+ if defaultRetentionDays <= 0 {
+ defaultRetentionDays = 30
+ }
+ var periods []int
+ if err := r.db.WithContext(ctx).Unscoped().Model(&types.DataSource{}).
+ Where("sync_log_retention_days > 0 AND sync_log_retention_days <> ?", defaultRetentionDays).
+ Distinct().Pluck("sync_log_retention_days", &periods).Error; err != nil {
return err
}
+ periods = append(periods, defaultRetentionDays)
+ now := time.Now().UTC()
+ sourceByID := "data_sources.id = sync_logs.data_source_id"
+ sourceMatch := sourceByID + " AND data_sources.tenant_id = sync_logs.tenant_id"
+ liveSource := r.db.WithContext(ctx).Model(&types.DataSource{}).Select("1").Where(sourceByID)
+ terminal := r.db.WithContext(ctx).
+ Where("status IN ?", []string{
+ types.SyncLogStatusSuccess, types.SyncLogStatusPartial, types.SyncLogStatusCanceled,
+ }).
+ Or("status = ? AND NOT EXISTS (?)", types.SyncLogStatusFailed, liveSource)
+ for _, days := range periods {
+ // A very large user policy must not overflow a time.Duration and turn
+ // into a recent cutoff. No valid log can predate year 1.
+ if days > now.Year()*366 {
+ continue
+ }
+ cutoff := now.AddDate(0, 0, -days)
+ if cutoff.Year() < 1 {
+ continue
+ }
+ sources := r.db.WithContext(ctx).Unscoped().Model(&types.DataSource{}).
+ Select("1")
+ policy := "EXISTS (?)"
+ if days == defaultRetentionDays {
+ // Also retire orphan logs after the default horizon instead of
+ // keeping them forever once their source has been removed.
+ policy = "NOT EXISTS (?)"
+ sources = sources.Where(sourceByID).
+ Where("sync_log_retention_days > 0 AND sync_log_retention_days <> ?", days)
+ } else {
+ sources = sources.Where(sourceMatch).
+ Where("sync_log_retention_days = ?", days)
+ }
+ if err := r.db.WithContext(ctx).
+ Where(terminal).
+ Where("COALESCE(finished_at, started_at) < ?", cutoff).
+ Where(policy, sources).
+ Delete(&types.SyncLog{}).Error; err != nil {
+ return err
+ }
+ }
return nil
}
diff --git a/internal/application/repository/datasource_retention_test.go b/internal/application/repository/datasource_retention_test.go
new file mode 100644
index 000000000..1b5ce2404
--- /dev/null
+++ b/internal/application/repository/datasource_retention_test.go
@@ -0,0 +1,120 @@
+package repository
+
+import (
+ "context"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/postgres"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+)
+
+func TestSyncLogRetentionPerSource(t *testing.T) {
+ for _, backend := range []string{"sqlite", "postgres"} {
+ t.Run(backend, func(t *testing.T) {
+ dialect := sqlite.Open(":memory:")
+ if backend == "postgres" {
+ dsn := os.Getenv("WEKNORA_REPOSITORY_TEST_POSTGRES_DSN")
+ if dsn == "" {
+ t.Skip("set WEKNORA_REPOSITORY_TEST_POSTGRES_DSN for PostgreSQL sync log retention tests")
+ }
+ dialect = postgres.Open(dsn)
+ }
+ db, err := gorm.Open(dialect, &gorm.Config{})
+ require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+ db = db.Begin()
+ require.NoError(t, db.Error)
+ t.Cleanup(func() { require.NoError(t, db.Rollback().Error) })
+ // Temporary tables exercise both dialects without modifying any
+ // existing deployment data when an integration-test DSN is supplied.
+ require.NoError(t, db.Exec(`CREATE TEMP TABLE data_sources (
+ id TEXT PRIMARY KEY, tenant_id BIGINT, sync_log_retention_days INT, deleted_at TIMESTAMP
+ )`).Error)
+ require.NoError(t, db.Exec(`CREATE TEMP TABLE sync_logs (
+ id TEXT PRIMARY KEY, data_source_id TEXT, tenant_id BIGINT, status TEXT,
+ started_at TIMESTAMP, finished_at TIMESTAMP
+ )`).Error)
+ now := time.Now().UTC()
+ for _, source := range []struct {
+ id string
+ tenant uint64
+ days int
+ deleted bool
+ }{
+ {"short", 1, 1, false},
+ {"long", 2, 30, false},
+ {"unrelated", 3, 90, false},
+ {"unset", 1, 0, false},
+ {"negative", 1, -1, false},
+ {"deleted-long", 2, 90, true},
+ {"deleted-short", 2, 1, true},
+ {"large", 1, 1 << 30, false},
+ } {
+ var deletedAt *time.Time
+ if source.deleted {
+ deletedAt = &now
+ }
+ require.NoError(t, db.Exec("INSERT INTO data_sources VALUES (?, ?, ?, ?)",
+ source.id, source.tenant, source.days, deletedAt).Error)
+ }
+ var want []string
+ for _, log := range []struct {
+ id string
+ source string
+ tenant uint64
+ status string
+ ageDays int
+ finishedAge int
+ keep bool
+ }{
+ {"short-expired", "short", 1, types.SyncLogStatusSuccess, 2, 2, false},
+ {"partial-expired", "short", 1, types.SyncLogStatusPartial, 2, 2, false},
+ {"canceled-expired", "short", 1, types.SyncLogStatusCanceled, 2, 2, false},
+ {"recent", "short", 1, types.SyncLogStatusSuccess, 0, 0, true},
+ {"recently-finished", "short", 1, types.SyncLogStatusSuccess, 60, 0, true},
+ {"legacy-expired", "short", 1, types.SyncLogStatusSuccess, 2, -1, false},
+ {"running", "short", 1, types.SyncLogStatusRunning, 60, -1, true},
+ {"pending", "short", 1, "pending", 60, -1, true},
+ {"retry", "short", 1, "retry", 60, -1, true},
+ {"failed-retryable", "short", 1, types.SyncLogStatusFailed, 60, 60, true},
+ {"long-retained", "long", 2, types.SyncLogStatusSuccess, 2, 2, true},
+ {"long-expired", "long", 2, types.SyncLogStatusSuccess, 31, 31, false},
+ {"unrelated-retained", "unrelated", 3, types.SyncLogStatusSuccess, 60, 60, true},
+ {"unset-expired", "unset", 1, types.SyncLogStatusSuccess, 31, 31, false},
+ {"negative-retained", "negative", 1, types.SyncLogStatusSuccess, 2, 2, true},
+ {"negative-expired", "negative", 1, types.SyncLogStatusSuccess, 31, 31, false},
+ {"deleted-policy-retained", "deleted-long", 2, types.SyncLogStatusSuccess, 60, 60, true},
+ {"deleted-policy-expired", "deleted-short", 2, types.SyncLogStatusFailed, 2, 2, false},
+ {"orphan-retained", "missing", 1, types.SyncLogStatusSuccess, 2, 2, true},
+ {"orphan-expired", "missing", 1, types.SyncLogStatusFailed, 31, 31, false},
+ {"tenant-mismatch-retained", "short", 9, types.SyncLogStatusSuccess, 60, 60, true},
+ {"large-policy-retained", "large", 1, types.SyncLogStatusSuccess, 60, 60, true},
+ } {
+ var finishedAt *time.Time
+ if log.finishedAge >= 0 {
+ finished := now.AddDate(0, 0, -log.finishedAge)
+ finishedAt = &finished
+ }
+ require.NoError(t, db.Exec("INSERT INTO sync_logs VALUES (?, ?, ?, ?, ?, ?)",
+ log.id, log.source, log.tenant, log.status, now.AddDate(0, 0, -log.ageDays), finishedAt).Error)
+ if log.keep {
+ want = append(want, log.id)
+ }
+ }
+ repo := NewSyncLogRepository(db)
+ for _, defaultDays := range []int{30, 0} {
+ require.NoError(t, repo.CleanupOldLogs(context.Background(), defaultDays))
+ var got []string
+ require.NoError(t, db.Table("sync_logs").Pluck("id", &got).Error)
+ require.ElementsMatch(t, want, got, "cleanup must be isolated and idempotent")
+ }
+ })
+ }
+}
diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.go
index fd5954458..c81e28430 100644
--- a/internal/datasource/scheduler.go
+++ b/internal/datasource/scheduler.go
@@ -85,12 +85,27 @@ func (s *Scheduler) Start(ctx context.Context) error {
ds.ID, ds.SyncSchedule, err)
}
}
+ // Retention also runs for paused and manual-only sources. Reuse this
+ // runner so StopWithin waits for cleanup along with in-flight sync jobs.
+ if _, err := s.cron.AddJob("@daily", cron.NewChain(
+ cron.SkipIfStillRunning(cron.DefaultLogger),
+ ).Then(cron.FuncJob(s.cleanupSyncLogs))); err != nil {
+ return fmt.Errorf("register sync log retention: %w", err)
+ }
s.cron.Start()
logger.Infof(ctx, "[Scheduler] started with %d cron entries", len(s.entries))
return nil
}
+func (s *Scheduler) cleanupSyncLogs() {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ if err := s.syncLogRepo.CleanupOldLogs(ctx, 30); err != nil {
+ logger.Warnf(ctx, "[Scheduler] sync log retention failed: %v", err)
+ }
+}
+
// Stop gracefully stops the cron runner and waits for running jobs to finish.
func (s *Scheduler) Stop() {
s.StopWithin(0)
diff --git a/internal/datasource/scheduler_retention_test.go b/internal/datasource/scheduler_retention_test.go
new file mode 100644
index 000000000..e42e5057f
--- /dev/null
+++ b/internal/datasource/scheduler_retention_test.go
@@ -0,0 +1,82 @@
+package datasource
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+type retentionCall struct {
+ ctx context.Context
+ days int
+}
+
+type retentionSyncLogRepo struct {
+ *fakeSyncLogRepo
+ calls chan retentionCall
+ release <-chan struct{}
+}
+
+func (r *retentionSyncLogRepo) CleanupOldLogs(ctx context.Context, days int) error {
+ r.calls <- retentionCall{ctx, days}
+ select {
+ case <-r.release:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func TestSchedulerRunsDailyRetentionAndWaitsForShutdown(t *testing.T) {
+ release := make(chan struct{})
+ var releaseOnce sync.Once
+ unblock := func() { releaseOnce.Do(func() { close(release) }) }
+ repo := &retentionSyncLogRepo{newFakeSyncLogRepo(), make(chan retentionCall, 1), release}
+ scheduler := NewScheduler(newFakeDataSourceRepo(), repo, &fakeTaskEnqueuer{})
+ require.NoError(t, scheduler.Start(context.Background()))
+ t.Cleanup(scheduler.Stop)
+ t.Cleanup(unblock)
+
+ // Retention must be registered even without a source cron entry, so
+ // paused/manual-only/deleted sources are also eventually cleaned.
+ require.Zero(t, scheduler.EntryCount())
+ entries := scheduler.cron.Entries()
+ require.Len(t, entries, 1)
+ now := time.Now()
+ want := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location())
+ require.Equal(t, want, entries[0].Schedule.Next(now))
+
+ // Run the registered production job through cron at a shorter test
+ // interval; cron's in-flight wait group must include the retention call.
+ scheduler.cron.Remove(entries[0].ID)
+ _, err := scheduler.cron.AddJob("@every 1s", entries[0].Job)
+ require.NoError(t, err)
+ select {
+ case call := <-repo.calls:
+ require.Equal(t, 30, call.days)
+ deadline, ok := call.ctx.Deadline()
+ require.True(t, ok, "retention database calls must have a timeout")
+ require.InDelta(t, 30, time.Until(deadline).Seconds(), 2)
+ case <-time.After(3 * time.Second):
+ t.Fatal("retention job did not run")
+ }
+ stopped := make(chan struct{})
+ go func() {
+ scheduler.Stop()
+ close(stopped)
+ }()
+ select {
+ case <-stopped:
+ t.Fatal("scheduler stopped before its retention call finished")
+ case <-time.After(20 * time.Millisecond):
+ }
+ unblock()
+ select {
+ case <-stopped:
+ case <-time.After(3 * time.Second):
+ t.Fatal("scheduler failed to stop after retention completed")
+ }
+}
diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go
index a24a05a06..e39da0171 100644
--- a/internal/types/interfaces/datasource.go
+++ b/internal/types/interfaces/datasource.go
@@ -135,6 +135,8 @@ type SyncLogRepository interface {
// CancelPendingByDataSource marks all non-terminal sync logs for a data source as canceled.
CancelPendingByDataSource(ctx context.Context, dsID string) error
- // CleanupOldLogs deletes sync logs older than the retention period
- CleanupOldLogs(ctx context.Context, retentionDays int) error
+ // CleanupOldLogs deletes expired terminal logs using each source's policy.
+ // Failed logs of live sources are retained because they may be awaiting retry.
+ // defaultRetentionDays applies to missing sources and unset policies.
+ CleanupOldLogs(ctx context.Context, defaultRetentionDays int) error
}
diff --git a/website-docs/04-api/02-api-infra.md b/website-docs/04-api/02-api-infra.md
index 9c61c6996..83956ce02 100644
--- a/website-docs/04-api/02-api-infra.md
+++ b/website-docs/04-api/02-api-infra.md
@@ -342,10 +342,12 @@ curl -X POST $BASE/api/v1/datasource/validate-credentials -H "Authorization: Bea
| `sync_mode` | string | 否 | `incremental`(默认)/`full` |
| `conflict_strategy` | string | 否 | `overwrite`(默认)/`skip` |
| `sync_deletions` | bool | 否 | 默认 true |
-| `sync_log_retention_days` | int | 否 | 默认 30 |
+| `sync_log_retention_days` | int | 否 | 本数据源同步日志保留天数,默认 30;非正值按 30 天处理 |
响应:201 `DataSourceResponse`(凭证剥离,见 `internal/handler/dto/datasource.go`)。
+后台每日凌晨(服务所在时区)清理过期的 `success`、`partial`、`canceled` 同步日志,包括暂停和仅手动同步的数据源。保留期从完成时间计算,旧日志缺少完成时间时使用开始时间。运行中或等待执行的日志不清理;现有数据源的 `failed` 日志也会保留,因为该状态可能仍在等待任务重试。数据源软删除后继续沿用原保留期,并可清理 `failed` 日志;无来源日志按默认 30 天保留。
+
```bash
curl -X POST $BASE/api/v1/datasource -H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' \