Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: rename export archive cleaner to generic data cleaner (#18433)
p0ny20c986367745
diff --git a/backend/runner/cleaner/data_cleaner.go b/backend/runner/cleaner/data_cleaner.gonew file mode 100644index 0000000000..d193853d8a--- /dev/null+++ b/backend/runner/cleaner/data_cleaner.go@@ -0,0 +1,90 @@+package cleaner++import (+ "context"+ "fmt"+ "log/slog"+ "sync"+ "time"++ "github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/store"+)++const (+ cleanupInterval = 1 * time.Hour+ exportArchiveRetentionPeriod = 24 * time.Hour+ oauth2ClientRetentionPeriod = 30 * 24 * time.Hour // 30 days of inactivity+)++// DataCleaner periodically cleans up expired data from the database.+type DataCleaner struct {+ store *store.Store+}++// NewDataCleaner creates a new DataCleaner.+func NewDataCleaner(store *store.Store) *DataCleaner {+ return &DataCleaner{+ store: store,+ }+}++// Run starts the DataCleaner.+func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {+ defer wg.Done()+ ticker := time.NewTicker(cleanupInterval)+ defer ticker.Stop()+ slog.Debug(fmt.Sprintf("Data cleaner started and will run every %v", cleanupInterval))++ // Run once immediately on startup+ c.cleanup(ctx)++ for {+ select {+ case <-ticker.C:+ c.cleanup(ctx)+ case <-ctx.Done():+ return+ }+ }+}++func (c *DataCleaner) cleanup(ctx context.Context) {+ c.cleanupExportArchives(ctx)+ c.cleanupOAuth2Data(ctx)+}++func (c *DataCleaner) cleanupExportArchives(ctx context.Context) {+ rowsAffected, err := c.store.DeleteExpiredExportArchives(ctx, exportArchiveRetentionPeriod)+ if err != nil {+ slog.Error("Failed to clean up expired export archives", log.BBError(err))+ return+ }+ if rowsAffected > 0 {+ slog.Info("Cleaned up expired export archives", slog.Int64("count", rowsAffected))+ }+}++func (c *DataCleaner) cleanupOAuth2Data(ctx context.Context) {+ // Clean up expired authorization codes+ if rowsAffected, err := c.store.DeleteExpiredOAuth2AuthorizationCodes(ctx); err != nil {+ slog.Error("Failed to clean up expired OAuth2 authorization codes", log.BBError(err))+ } else if rowsAffected > 0 {+ slog.Info("Cleaned up expired OAuth2 authorization codes", slog.Int64("count", rowsAffected))+ }++ // Clean up expired refresh tokens+ if rowsAffected, err := c.store.DeleteExpiredOAuth2RefreshTokens(ctx); err != nil {+ slog.Error("Failed to clean up expired OAuth2 refresh tokens", log.BBError(err))+ } else if rowsAffected > 0 {+ slog.Info("Cleaned up expired OAuth2 refresh tokens", slog.Int64("count", rowsAffected))+ }++ // Clean up inactive OAuth2 clients (DCR clients that haven't been used)+ expireBefore := time.Now().Add(-oauth2ClientRetentionPeriod)+ if rowsAffected, err := c.store.DeleteExpiredOAuth2Clients(ctx, expireBefore); err != nil {+ slog.Error("Failed to clean up inactive OAuth2 clients", log.BBError(err))+ } else if rowsAffected > 0 {+ slog.Info("Cleaned up inactive OAuth2 clients", slog.Int64("count", rowsAffected))+ }+}diff --git a/backend/runner/migrator/export_archive_cleaner.go b/backend/runner/migrator/export_archive_cleaner.godeleted file mode 100644index 119df26df1..0000000000--- a/backend/runner/migrator/export_archive_cleaner.go+++ /dev/null@@ -1,67 +0,0 @@-package migrator--import (- "context"- "fmt"- "log/slog"- "sync"- "time"-- "github.com/bytebase/bytebase/backend/common/log"- "github.com/bytebase/bytebase/backend/store"-)--const (- exportArchiveCleanupInterval = 1 * time.Hour- exportArchiveRetentionPeriod = 24 * time.Hour-)--// ExportArchiveCleaner is the cleaner for expired export archives.-// It runs periodically to delete export archives older than the retention period.-type ExportArchiveCleaner struct {- store *store.Store-}--// NewExportArchiveCleaner creates a new ExportArchiveCleaner.-func NewExportArchiveCleaner(store *store.Store) *ExportArchiveCleaner {- return &ExportArchiveCleaner{- store: store,- }-}--// Run starts the ExportArchiveCleaner.-func (c *ExportArchiveCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {- defer wg.Done()- ticker := time.NewTicker(exportArchiveCleanupInterval)- defer ticker.Stop()- slog.Debug(fmt.Sprintf("Export archive cleaner started and will run every %v", exportArchiveCleanupInterval))-- // Run once immediately on startup- if err := c.cleanup(ctx); err != nil {- slog.Error("Failed to run export archive cleanup on startup", log.BBError(err))- }-- for {- select {- case <-ticker.C:- if err := c.cleanup(ctx); err != nil {- slog.Error("Failed to run export archive cleanup", log.BBError(err))- }- case <-ctx.Done():- return- }- }-}--func (c *ExportArchiveCleaner) cleanup(ctx context.Context) error {- rowsAffected, err := c.store.DeleteExpiredExportArchives(ctx, exportArchiveRetentionPeriod)- if err != nil {- return err- }-- if rowsAffected > 0 {- slog.Info("Cleaned up expired export archives", slog.Int64("count", rowsAffected))- }-- return nil-}diff --git a/backend/server/server.go b/backend/server/server.goindex 22891d93cc..6e7ea40a0c 100644--- a/backend/server/server.go+++ b/backend/server/server.go@@ -37,8 +37,8 @@ import ("github.com/bytebase/bytebase/backend/migrator""github.com/bytebase/bytebase/backend/resources/postgres""github.com/bytebase/bytebase/backend/runner/approval"+ "github.com/bytebase/bytebase/backend/runner/cleaner""github.com/bytebase/bytebase/backend/runner/metricreport"- runnermigrator "github.com/bytebase/bytebase/backend/runner/migrator""github.com/bytebase/bytebase/backend/runner/monitor""github.com/bytebase/bytebase/backend/runner/plancheck""github.com/bytebase/bytebase/backend/runner/schemasync"@@ -58,13 +58,13 @@ const (// Server is the Bytebase server.type Server struct {// Asynchronous runners.- taskScheduler *taskrun.Scheduler- planCheckScheduler *plancheck.Scheduler- metricReporter *metricreport.Reporter- schemaSyncer *schemasync.Syncer- approvalRunner *approval.Runner- exportArchiveCleaner *runnermigrator.ExportArchiveCleaner- runnerWG sync.WaitGroup+ taskScheduler *taskrun.Scheduler+ planCheckScheduler *plancheck.Scheduler+ metricReporter *metricreport.Reporter+ schemaSyncer *schemasync.Syncer+ approvalRunner *approval.Runner+ dataCleaner *cleaner.DataCleaner+ runnerWG sync.WaitGroupwebhookManager *webhook.ManageriamManager *iam.Manager@@ -208,8 +208,8 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {statementReportExecutor := plancheck.NewStatementReportExecutor(stores, sheetManager, s.dbFactory)s.planCheckScheduler.Register(store.PlanCheckDatabaseStatementSummaryReport, statementReportExecutor)- // Export archive cleaner- s.exportArchiveCleaner = runnermigrator.NewExportArchiveCleaner(stores)+ // Data cleaner+ s.dataCleaner = cleaner.NewDataCleaner(stores)// Metric reporters.initMetricReporter()@@ -249,7 +249,7 @@ func (s *Server) Run(ctx context.Context, port int) error {go s.planCheckScheduler.Run(ctx, &s.runnerWG)s.runnerWG.Add(1)- go s.exportArchiveCleaner.Run(ctx, &s.runnerWG)+ go s.dataCleaner.Run(ctx, &s.runnerWG)s.runnerWG.Add(1)mmm := monitor.NewMemoryMonitor(s.profile)