Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: automatically clean up upgrade logs with configurable retention (#4230)
Kyle Mendell55ccb321037c
diff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.goindex f0c0c30d6..6f726a122 100644--- a/backend/cli/upgrade/upgrade.go+++ b/backend/cli/upgrade/upgrade.go@@ -12,6 +12,7 @@ import ("emperror.dev/errors"docker "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/network"@@ -60,7 +61,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error {// This prevents interruption when stopping the target containerctx := context.Background()- logFile, err := SetupMessageOnlyLogFile("/app/data", "arcane-upgrade", slog.LevelInfo)+ logFile, err := SetupMessageOnlyLogFile(libarcane.UpgradeLogDirectory, "arcane-upgrade", slog.LevelInfo)if err != nil {slog.Warn("Failed to setup file logging", "error", err)} else {diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex b38ed1939..5993064fc 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -96,6 +96,7 @@ type registerJobsParams struct {ImageUpdateWatcher *scheduler.ImageUpdateWatcherDockerClientRefresh *scheduler.DockerClientRefreshJobAnalytics *scheduler.AnalyticsJob+ UpgradeLogCleanup schedulertypes.Job `name:"upgrade-log-cleanup"`EventCleanup *scheduler.EventCleanupJobPruningVolumeHelper *scheduler.PruningVolumeHelperJobExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJob@@ -156,6 +157,7 @@ func registerJobs(params registerJobsParams) error {params.ActivitySweep,params.UploadSessionsCleanup,params.GitCloneCleanup,+ params.UpgradeLogCleanup,params.ApnsOutbox,} {if err := params.Scheduler.RegisterJob(job); err != nil {diff --git a/backend/internal/config/schema/schema_test.go b/backend/internal/config/schema/schema_test.goindex ce4b1091a..422cd4c3c 100644--- a/backend/internal/config/schema/schema_test.go+++ b/backend/internal/config/schema/schema_test.go@@ -378,6 +378,7 @@ var expectedSettingOverrideKeys = []string{"trivyServerToken","trivyServerUrl","updateCheckRegistry",+ "upgradeLogRetentionDays","volumeHelperIdleTimeout","vulnerabilityScanEnabled","vulnerabilityScanInterval",diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex 61f653328..ba1593cde 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -162,6 +162,7 @@ var JobOptions = fx.Options(scheduler.NewActivitySweepJob,scheduler.NewUploadSessionsCleanupJob,scheduler.NewGitCloneCleanupJob,+ fx.Annotate(scheduler.NewUpgradeLogCleanupJob, fx.ResultTags(`name:"upgrade-log-cleanup"`)),scheduler.NewApnsOutboxJob,),)diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex c3703b5ac..da2101098 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -299,7 +299,7 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponseenabled := s.isJobEnabledInternal(ctx, jobMeta)prerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)- if s.scheduler != nil && jobMeta.SettingsKey != "" && jobMeta.ID != "environment-health" {+ if s.scheduler != nil && (jobMeta.SettingsKey != "" || jobMeta.ID == "upgrade-log-cleanup") && jobMeta.ID != "environment-health" {if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok {schedule = cmp.Or(runtimeState.Schedule, schedule)nextRun = runtimeState.NextRundiff --git a/backend/internal/settings/features.go b/backend/internal/settings/features.goindex c9bf54d54..735b20610 100644--- a/backend/internal/settings/features.go+++ b/backend/internal/settings/features.go@@ -35,12 +35,3 @@ func (s *SettingsService) RequireFeature(ctx context.Context, id features.ID) er}return errors.WrapIff(common.ErrFeatureDisabled, "feature %s is disabled", id)}--func validateFeatureSettingInternal(key, value string) error {- for _, definition := range features.All() {- if definition.SettingKey == key && value != "true" && value != "false" {- return common.Classify(common.ErrValidation, errors.Errorf("%s must be true or false", key))- }- }- return nil-}diff --git a/backend/internal/settings/model.go b/backend/internal/settings/model.goindex 5b25e387f..f6252c0a0 100644--- a/backend/internal/settings/model.go+++ b/backend/internal/settings/model.go@@ -85,6 +85,7 @@ type Settings struct {EventCleanupInterval SettingVariable `key:"eventCleanupInterval" meta:"label=Event Cleanup Interval;type=cron;keywords=events,cleanup,retention,interval,frequency,schedule,history,logs,jobs;description=How often to delete old events (cron expression)"`ExpiredSessionsCleanupInterval SettingVariable `key:"expiredSessionsCleanupInterval" meta:"label=Expired Sessions Cleanup Interval;type=cron;keywords=sessions,cleanup,retention,expired,revoked,interval,frequency,schedule,auth,jobs;description=How often to delete expired and old revoked sessions (cron expression)"`ActivityHistoryRetentionDays SettingVariable `key:"activityHistoryRetentionDays" meta:"label=Activity History Retention;type=number;keywords=activity,history,retention,days,cleanup,background,tasks;category=activity;description=Delete completed Activity Center entries older than this many days. Set 0 to disable age-based cleanup." catmeta:"id=activity;title=Activity;icon=activity;url=/settings/activity;description=Configure Activity Center history and cleanup"`+ UpgradeLogRetentionDays SettingVariable `key:"upgradeLogRetentionDays" meta:"label=Upgrade Log Retention;type=number;keywords=upgrade,logs,retention,days,cleanup;category=activity;description=Delete upgrade log files older than this many days. Cleanup runs hourly. Default: 3 days. Set 0 to keep logs indefinitely."`ActivityHistoryMaxEntries SettingVariable `key:"activityHistoryMaxEntries" meta:"label=Activity History Limit;type=number;keywords=activity,history,limit,entries,count,cleanup,background,tasks;category=activity;description=Maximum completed Activity Center entries to keep per environment. Set 0 to disable count-based cleanup."`MaxConcurrentActivities SettingVariable `key:"maxConcurrentActivities" meta:"label=Concurrent Activity Limit;type=number;keywords=activity,concurrency,limit,queue,parallel,bulk,background,tasks;category=activity;description=Maximum long-running activities per environment before new ones queue. Set 0 for unlimited."`AutoInjectEnv SettingVariable `key:"autoInjectEnv" meta:"label=Auto Inject Env Variables;type=boolean;keywords=auto,inject,env,environment,variables,interpolation;category=internal;description=Automatically inject project .env variables into all containers (default: false)"`diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.goindex 15213b068..c7e90ab37 100644--- a/backend/internal/settings/service.go+++ b/backend/internal/settings/service.go@@ -29,6 +29,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"+ "github.com/getarcaneapp/arcane/types/v2/features"settingstypes "github.com/getarcaneapp/arcane/types/v2/settings""github.com/samber/mo""go.getarcane.app/kit/normalization"@@ -211,6 +212,7 @@ func DefaultSettingsConfig() *Settings {EventCleanupInterval: SettingVariable{Value: "0 0 */6 * * *"},ExpiredSessionsCleanupInterval: SettingVariable{Value: "0 0 0 * * *"},ActivityHistoryRetentionDays: SettingVariable{Value: "30"},+ UpgradeLogRetentionDays: SettingVariable{Value: "3"},ActivityHistoryMaxEntries: SettingVariable{Value: "1000"},MaxConcurrentActivities: SettingVariable{Value: "5"},AutoInjectEnv: SettingVariable{Value: "false"},@@ -495,8 +497,23 @@ func (s *SettingsService) getEffectiveSettingsConfigInternal(ctx context.Contextreturn settingsCfg}+func validateSettingValueInternal(key, value string) error {+ if key == "upgradeLogRetentionDays" && value != "" {+ days, err := strconv.Atoi(value)+ if err != nil || days < 0 || days > 3650 {+ return common.Classify(common.ErrValidation, errors.New("upgradeLogRetentionDays must be a whole number between 0 and 3650"))+ }+ }+ for _, definition := range features.All() {+ if definition.SettingKey == key && value != "true" && value != "false" {+ return common.Classify(common.ErrValidation, errors.Errorf("%s must be true or false", key))+ }+ }+ return nil+}+func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {- if err := validateFeatureSettingInternal(key, value); err != nil {+ if err := validateSettingValueInternal(key, value); err != nil {return err}if err := libarcane.ValidateCronSetting(key, value); err != nil {@@ -516,7 +533,7 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libarcane.SettingUpdate) error {values := make([]SettingVariable, 0, len(updates))for _, update := range updates {- if err := validateFeatureSettingInternal(update.Key, update.Value); err != nil {+ if err := validateSettingValueInternal(update.Key, update.Value); err != nil {return err}values = append(values, SettingVariable{Key: update.Key, Value: update.Value})@@ -678,7 +695,7 @@ func (s *SettingsService) prepareUpdateValues(updates settingstypes.Update, cfg,continue}- if err := validateFeatureSettingInternal(key, value); err != nil {+ if err := validateSettingValueInternal(key, value); err != nil {return nil, err}diff --git a/backend/internal/system/upgrade_service.go b/backend/internal/system/upgrade_service.goindex 017179db3..32fee61c3 100644--- a/backend/internal/system/upgrade_service.go+++ b/backend/internal/system/upgrade_service.go@@ -6,9 +6,11 @@ import (stderrors "errors""fmt""io"+ "io/fs""log/slog""net/http""os"+ "regexp""slices""strings""sync/atomic"@@ -44,6 +46,8 @@ import ("golang.org/x/mod/semver")+var upgradeLogNameInternal = regexp.MustCompile(`^arcane-upgrade-[0-9]+\.log$`)+type SystemUpgradeService struct {upgrading atomic.BoolupdatingAll atomic.Bool@@ -242,7 +246,7 @@ func (s *SystemUpgradeService) runPreparedUpgradeInternal(ctx context.Context, pslog.Info("Upgrader image pulled successfully", "image", upgraderImage)// Try to get the /app/data mount from current container so upgrade logs persist.- appDataMount := dockerutils.MountForDestination(prepared.current.Mounts, "/app/data", "/app/data")+ appDataMount := dockerutils.MountForDestination(prepared.current.Mounts, libarcane.UpgradeLogDirectory, libarcane.UpgradeLogDirectory)if appDataMount == nil {slog.Warn("Could not detect /app/data mount; upgrader logs may not persist")} else {@@ -1346,3 +1350,61 @@ func truncateUpdateAllErrorInternal(err error) string {}return msg}++// PruneUpgradeLogs removes expired upgrade logs from this instance's data directory.+func (s *SystemUpgradeService) PruneUpgradeLogs(ctx context.Context, dataDir string, now time.Time) (int, error) {+ retentionDays := s.settingsService.GetIntSetting(ctx, "upgradeLogRetentionDays", 3)+ if retentionDays == 0 {+ return 0, nil+ }+ if retentionDays < 0 || retentionDays > 3650 {+ return 0, errors.New("upgrade log retention must be between 0 and 3650 days")+ }+ if err := ctx.Err(); err != nil {+ return 0, err+ }++ root, err := os.OpenRoot(dataDir)+ if errors.Is(err, os.ErrNotExist) {+ return 0, nil+ }+ if err != nil {+ return 0, errors.WrapIf(err, "open upgrade log directory")+ }+ defer root.Close()+ entries, err := fs.ReadDir(root.FS(), ".")+ if err != nil {+ return 0, errors.WrapIf(err, "read upgrade log directory")+ }++ cutoff := now.Add(-time.Duration(retentionDays) * 24 * time.Hour)+ removed := 0+ var failures []error+ for _, entry := range entries {+ if err := ctx.Err(); err != nil {+ return removed, stderrors.Join(append(failures, err)...)+ }+ if !upgradeLogNameInternal.MatchString(entry.Name()) {+ continue+ }+ info, err := root.Lstat(entry.Name())+ if errors.Is(err, os.ErrNotExist) {+ continue+ }+ if err != nil {+ failures = append(failures, errors.WrapIff(err, "inspect upgrade log %s", entry.Name()))+ continue+ }+ if !info.Mode().IsRegular() || !info.ModTime().Before(cutoff) {+ continue+ }+ if err := root.Remove(entry.Name()); err != nil {+ if !errors.Is(err, os.ErrNotExist) {+ failures = append(failures, errors.WrapIff(err, "remove upgrade log %s", entry.Name()))+ }+ continue+ }+ removed+++ }+ return removed, stderrors.Join(failures...)+}diff --git a/backend/pkg/libarcane/internal_resource.go b/backend/pkg/libarcane/internal_resource.goindex 82f49e905..cf5fd8c1c 100644--- a/backend/pkg/libarcane/internal_resource.go+++ b/backend/pkg/libarcane/internal_resource.go@@ -16,6 +16,8 @@ import ()const (+ // UpgradeLogDirectory is mounted into the upgrader to persist its logs.+ UpgradeLogDirectory = "/app/data"// InternalResourceLabel marks containers used for Arcane utilities.InternalResourceLabel = "com.getarcaneapp.internal.resource"// HiddenResourceLabel hides containers from container lists and dashboard counts.diff --git a/backend/pkg/scheduler/upgrade_log_cleanup_job.go b/backend/pkg/scheduler/upgrade_log_cleanup_job.gonew file mode 100644index 000000000..68625169c--- /dev/null+++ b/backend/pkg/scheduler/upgrade_log_cleanup_job.go@@ -0,0 +1,32 @@+package scheduler++import (+ "context"+ "log/slog"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/system"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++const UpgradeLogCleanupJobName = "upgrade-log-cleanup"++// NewUpgradeLogCleanupJob creates the hourly local upgrade log cleanup job.+func NewUpgradeLogCleanupJob(service *system.SystemUpgradeService) schedulertypes.Job {+ return &schedulertypes.GenericJob{+ JobName: UpgradeLogCleanupJobName,+ ScheduleFn: func(context.Context) string { return "0 15 * * * *" },+ RunFn: func(ctx context.Context) (schedulertypes.Outcome, error) {+ removed, err := service.PruneUpgradeLogs(ctx, libarcane.UpgradeLogDirectory, time.Now())+ if err != nil {+ slog.ErrorContext(ctx, "Failed to clean up upgrade logs", "jobName", UpgradeLogCleanupJobName, "removed", removed, "error", err)+ return schedulertypes.Outcome{}, err+ }+ if removed > 0 {+ slog.InfoContext(ctx, "Removed expired upgrade logs", "jobName", UpgradeLogCleanupJobName, "removed", removed)+ }+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil+ },+ }+}diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 02ea53ad7..32ed75839 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -2324,6 +2324,10 @@"activity_cancel_already_finished": "Activity already finished","activity_settings_description": "Configure Activity Center history retention for each environment.","activity_settings_saved": "Activity settings saved",+ "upgrade_logs_section_title": "Upgrade logs",+ "upgrade_log_retention_days": "Upgrade log retention (days)",+ "upgrade_log_retention_days_description": "Delete upgrade log files older than this many days. Defaults to 3 days.",+ "upgrade_log_retention_days_help": "Cleanup runs hourly for this environment. Set 0 to keep logs indefinitely. Changes apply on the next cleanup run.","activity_history_section_title": "History Retention","activity_history_retention_days": "Retention Days","activity_history_retention_days_description": "Delete completed Activity Center entries older than this many days.",diff --git a/frontend/src/lib/services/settings-service.ts b/frontend/src/lib/services/settings-service.tsindex 75d7deb62..6ce81f5cf 100644--- a/frontend/src/lib/services/settings-service.ts+++ b/frontend/src/lib/services/settings-service.ts@@ -138,7 +138,7 @@ class SettingsService extends BaseAPIService {}if (value === 'true') return true;if (value === 'false') return false;- if (key === 'projectWorkspaceMaxFileSizeMb' || key === 'volumeWorkspaceMaxFileSizeMb') {+ if (key === 'projectWorkspaceMaxFileSizeMb' || key === 'volumeWorkspaceMaxFileSizeMb' || key === 'upgradeLogRetentionDays') {return Number(value);}return value;diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.tsindex 47ad27daf..3c4efb55d 100644--- a/frontend/src/lib/types/settings.ts+++ b/frontend/src/lib/types/settings.ts@@ -30,6 +30,7 @@ export type Settings = {dockerClientRefreshInterval?: string;environmentHealthInterval: number;activityHistoryRetentionDays: number;+ upgradeLogRetentionDays: number;activityHistoryMaxEntries: number;maxConcurrentActivities: number;defaultDeployPullPolicy: 'missing' | 'always' | 'never';diff --git a/frontend/src/routes/(app)/settings/activity/+page.svelte b/frontend/src/routes/(app)/settings/activity/+page.svelteindex c9259d550..bf9700fdb 100644--- a/frontend/src/routes/(app)/settings/activity/+page.svelte+++ b/frontend/src/routes/(app)/settings/activity/+page.svelte@@ -13,6 +13,7 @@const formSchema = z.object({activityHistoryRetentionDays: z.coerce.number().int().min(0).max(3650),+ upgradeLogRetentionDays: z.coerce.number().int().min(0).max(3650),activityHistoryMaxEntries: z.coerce.number().int().min(0).max(100000),maxConcurrentActivities: z.coerce.number().int().min(0).max(1000)});@@ -21,6 +22,7 @@const settings = settingsStore.current || data.settings!;return {activityHistoryRetentionDays: settings.activityHistoryRetentionDays,+ upgradeLogRetentionDays: settings.upgradeLogRetentionDays ?? 3,activityHistoryMaxEntries: settings.activityHistoryMaxEntries,maxConcurrentActivities: settings.maxConcurrentActivities};@@ -75,6 +77,19 @@/></div></div>+ <div class="space-y-4">+ <h3 class="text-base font-semibold">{m.upgrade_logs_section_title()}</h3>+ <div class="grid gap-5 sm:grid-cols-2 lg:grid-cols-3">+ <TextInputWithLabel+ bind:value={formInputs.upgradeLogRetentionDays.value}+ error={formInputs.upgradeLogRetentionDays.error}+ label={m.upgrade_log_retention_days()}+ description={m.upgrade_log_retention_days_description()}+ helpText={m.upgrade_log_retention_days_help()}+ type="number"+ />+ </div>+ </div></fieldset>{/snippet}</SettingsPageLayout>diff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.goindex fec508f40..6e0569af8 100644--- a/types/meta/job_metadata.go+++ b/types/meta/job_metadata.go@@ -59,6 +59,16 @@ var jobMetadataRegistry = map[string]JobMetadata{CanRunManually: true,Prerequisites: []JobPrerequisiteMetadata{},},+ "upgrade-log-cleanup": {+ ID: "upgrade-log-cleanup",+ Name: "Upgrade Log Cleanup",+ Description: "Removes upgrade logs older than the configured retention period",+ Category: "maintenance",+ ManagerOnly: false,+ IsContinuous: false,+ CanRunManually: true,+ Prerequisites: []JobPrerequisiteMetadata{},+ },"expired-sessions-cleanup": {ID: "expired-sessions-cleanup",Name: "Expired Sessions Cleanup",diff --git a/types/settings/settings.go b/types/settings/settings.goindex e6173fc8d..b937d8e5a 100644--- a/types/settings/settings.go+++ b/types/settings/settings.go@@ -104,6 +104,11 @@ type Update struct {// Required: falseActivityHistoryRetentionDays *string `json:"activityHistoryRetentionDays,omitzero"`+ // UpgradeLogRetentionDays is the upgrade log retention in days. Zero disables cleanup.+ //+ // Required: false+ UpgradeLogRetentionDays *string `json:"upgradeLogRetentionDays,omitzero"`+// ActivityHistoryMaxEntries is the maximum completed Activity Center entries to retain per environment.//// Required: false