Skip to content

Modern Docker Management, Designed for Everyone

fix: automatically clean up upgrade logs with configurable retention (#4230)

Kyle Mendell55ccb321037c
diff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.go
index f0c0c30d6..6f726a122 100644
--- a/backend/cli/upgrade/upgrade.go
+++ b/backend/cli/upgrade/upgrade.go
@@ -12,6 +12,7 @@ import (
"emperror.dev/errors"
docker "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
@@ -60,7 +61,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error {
// This prevents interruption when stopping the target container
ctx := context.Background()
- logFile, err := SetupMessageOnlyLogFile("/app/data", "arcane-upgrade", slog.LevelInfo)
+ logFile, err := SetupMessageOnlyLogFile(libarcane.UpgradeLogDirectory, "arcane-upgrade", slog.LevelInfo)
if err != nil {
slog.Warn("Failed to setup file logging", "error", err)
} else {
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index b38ed1939..5993064fc 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -96,6 +96,7 @@ type registerJobsParams struct {
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
DockerClientRefresh *scheduler.DockerClientRefreshJob
Analytics *scheduler.AnalyticsJob
+ UpgradeLogCleanup schedulertypes.Job `name:"upgrade-log-cleanup"`
EventCleanup *scheduler.EventCleanupJob
PruningVolumeHelper *scheduler.PruningVolumeHelperJob
ExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJob
@@ -156,6 +157,7 @@ func registerJobs(params registerJobsParams) error {
params.ActivitySweep,
params.UploadSessionsCleanup,
params.GitCloneCleanup,
+ params.UpgradeLogCleanup,
params.ApnsOutbox,
} {
if err := params.Scheduler.RegisterJob(job); err != nil {
diff --git a/backend/internal/config/schema/schema_test.go b/backend/internal/config/schema/schema_test.go
index ce4b1091a..422cd4c3c 100644
--- a/backend/internal/config/schema/schema_test.go
+++ b/backend/internal/config/schema/schema_test.go
@@ -378,6 +378,7 @@ var expectedSettingOverrideKeys = []string{
"trivyServerToken",
"trivyServerUrl",
"updateCheckRegistry",
+ "upgradeLogRetentionDays",
"volumeHelperIdleTimeout",
"vulnerabilityScanEnabled",
"vulnerabilityScanInterval",
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index 61f653328..ba1593cde 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -162,6 +162,7 @@ var JobOptions = fx.Options(
scheduler.NewActivitySweepJob,
scheduler.NewUploadSessionsCleanupJob,
scheduler.NewGitCloneCleanupJob,
+ fx.Annotate(scheduler.NewUpgradeLogCleanupJob, fx.ResultTags(`name:"upgrade-log-cleanup"`)),
scheduler.NewApnsOutboxJob,
),
)
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.go
index c3703b5ac..da2101098 100644
--- a/backend/internal/job/service.go
+++ b/backend/internal/job/service.go
@@ -299,7 +299,7 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse
enabled := s.isJobEnabledInternal(ctx, jobMeta)
prerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)
- if s.scheduler != nil && jobMeta.SettingsKey != "" && jobMeta.ID != "environment-health" {
+ if s.scheduler != nil && (jobMeta.SettingsKey != "" || jobMeta.ID == "upgrade-log-cleanup") && jobMeta.ID != "environment-health" {
if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok {
schedule = cmp.Or(runtimeState.Schedule, schedule)
nextRun = runtimeState.NextRun
diff --git a/backend/internal/settings/features.go b/backend/internal/settings/features.go
index c9bf54d54..735b20610 100644
--- a/backend/internal/settings/features.go
+++ b/backend/internal/settings/features.go
@@ -35,12 +35,3 @@ func (s *SettingsService) RequireFeature(ctx context.Context, id features.ID) er
}
return errors.WrapIff(common.ErrFeatureDisabled, "feature %s is disabled", id)
}
-
-func validateFeatureSettingInternal(key, value string) error {
- for _, definition := range features.All() {
- if definition.SettingKey == key && value != "true" && value != "false" {
- return common.Classify(common.ErrValidation, errors.Errorf("%s must be true or false", key))
- }
- }
- return nil
-}
diff --git a/backend/internal/settings/model.go b/backend/internal/settings/model.go
index 5b25e387f..f6252c0a0 100644
--- a/backend/internal/settings/model.go
+++ b/backend/internal/settings/model.go
@@ -85,6 +85,7 @@ type Settings struct {
EventCleanupInterval SettingVariable `key:"eventCleanupInterval" meta:"label=Event Cleanup Interval;type=cron;keywords=events,cleanup,retention,interval,frequency,schedule,history,logs,jobs;description=How often to delete old events (cron expression)"`
ExpiredSessionsCleanupInterval SettingVariable `key:"expiredSessionsCleanupInterval" meta:"label=Expired Sessions Cleanup Interval;type=cron;keywords=sessions,cleanup,retention,expired,revoked,interval,frequency,schedule,auth,jobs;description=How often to delete expired and old revoked sessions (cron expression)"`
ActivityHistoryRetentionDays SettingVariable `key:"activityHistoryRetentionDays" meta:"label=Activity History Retention;type=number;keywords=activity,history,retention,days,cleanup,background,tasks;category=activity;description=Delete completed Activity Center entries older than this many days. Set 0 to disable age-based cleanup." catmeta:"id=activity;title=Activity;icon=activity;url=/settings/activity;description=Configure Activity Center history and cleanup"`
+ UpgradeLogRetentionDays SettingVariable `key:"upgradeLogRetentionDays" meta:"label=Upgrade Log Retention;type=number;keywords=upgrade,logs,retention,days,cleanup;category=activity;description=Delete upgrade log files older than this many days. Cleanup runs hourly. Default: 3 days. Set 0 to keep logs indefinitely."`
ActivityHistoryMaxEntries SettingVariable `key:"activityHistoryMaxEntries" meta:"label=Activity History Limit;type=number;keywords=activity,history,limit,entries,count,cleanup,background,tasks;category=activity;description=Maximum completed Activity Center entries to keep per environment. Set 0 to disable count-based cleanup."`
MaxConcurrentActivities SettingVariable `key:"maxConcurrentActivities" meta:"label=Concurrent Activity Limit;type=number;keywords=activity,concurrency,limit,queue,parallel,bulk,background,tasks;category=activity;description=Maximum long-running activities per environment before new ones queue. Set 0 for unlimited."`
AutoInjectEnv SettingVariable `key:"autoInjectEnv" meta:"label=Auto Inject Env Variables;type=boolean;keywords=auto,inject,env,environment,variables,interpolation;category=internal;description=Automatically inject project .env variables into all containers (default: false)"`
diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.go
index 15213b068..c7e90ab37 100644
--- a/backend/internal/settings/service.go
+++ b/backend/internal/settings/service.go
@@ -29,6 +29,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
+ "github.com/getarcaneapp/arcane/types/v2/features"
settingstypes "github.com/getarcaneapp/arcane/types/v2/settings"
"github.com/samber/mo"
"go.getarcane.app/kit/normalization"
@@ -211,6 +212,7 @@ func DefaultSettingsConfig() *Settings {
EventCleanupInterval: SettingVariable{Value: "0 0 */6 * * *"},
ExpiredSessionsCleanupInterval: SettingVariable{Value: "0 0 0 * * *"},
ActivityHistoryRetentionDays: SettingVariable{Value: "30"},
+ UpgradeLogRetentionDays: SettingVariable{Value: "3"},
ActivityHistoryMaxEntries: SettingVariable{Value: "1000"},
MaxConcurrentActivities: SettingVariable{Value: "5"},
AutoInjectEnv: SettingVariable{Value: "false"},
@@ -495,8 +497,23 @@ func (s *SettingsService) getEffectiveSettingsConfigInternal(ctx context.Context
return settingsCfg
}
+func validateSettingValueInternal(key, value string) error {
+ if key == "upgradeLogRetentionDays" && value != "" {
+ days, err := strconv.Atoi(value)
+ if err != nil || days < 0 || days > 3650 {
+ return common.Classify(common.ErrValidation, errors.New("upgradeLogRetentionDays must be a whole number between 0 and 3650"))
+ }
+ }
+ for _, definition := range features.All() {
+ if definition.SettingKey == key && value != "true" && value != "false" {
+ return common.Classify(common.ErrValidation, errors.Errorf("%s must be true or false", key))
+ }
+ }
+ return nil
+}
+
func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {
- if err := validateFeatureSettingInternal(key, value); err != nil {
+ if err := validateSettingValueInternal(key, value); err != nil {
return err
}
if err := libarcane.ValidateCronSetting(key, value); err != nil {
@@ -516,7 +533,7 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)
func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libarcane.SettingUpdate) error {
values := make([]SettingVariable, 0, len(updates))
for _, update := range updates {
- if err := validateFeatureSettingInternal(update.Key, update.Value); err != nil {
+ if err := validateSettingValueInternal(update.Key, update.Value); err != nil {
return err
}
values = append(values, SettingVariable{Key: update.Key, Value: update.Value})
@@ -678,7 +695,7 @@ func (s *SettingsService) prepareUpdateValues(updates settingstypes.Update, cfg,
continue
}
- if err := validateFeatureSettingInternal(key, value); err != nil {
+ if err := validateSettingValueInternal(key, value); err != nil {
return nil, err
}
diff --git a/backend/internal/system/upgrade_service.go b/backend/internal/system/upgrade_service.go
index 017179db3..32fee61c3 100644
--- a/backend/internal/system/upgrade_service.go
+++ b/backend/internal/system/upgrade_service.go
@@ -6,9 +6,11 @@ import (
stderrors "errors"
"fmt"
"io"
+ "io/fs"
"log/slog"
"net/http"
"os"
+ "regexp"
"slices"
"strings"
"sync/atomic"
@@ -44,6 +46,8 @@ import (
"golang.org/x/mod/semver"
)
+var upgradeLogNameInternal = regexp.MustCompile(`^arcane-upgrade-[0-9]+\.log$`)
+
type SystemUpgradeService struct {
upgrading atomic.Bool
updatingAll atomic.Bool
@@ -242,7 +246,7 @@ func (s *SystemUpgradeService) runPreparedUpgradeInternal(ctx context.Context, p
slog.Info("Upgrader image pulled successfully", "image", upgraderImage)
// Try to get the /app/data mount from current container so upgrade logs persist.
- appDataMount := dockerutils.MountForDestination(prepared.current.Mounts, "/app/data", "/app/data")
+ appDataMount := dockerutils.MountForDestination(prepared.current.Mounts, libarcane.UpgradeLogDirectory, libarcane.UpgradeLogDirectory)
if appDataMount == nil {
slog.Warn("Could not detect /app/data mount; upgrader logs may not persist")
} else {
@@ -1346,3 +1350,61 @@ func truncateUpdateAllErrorInternal(err error) string {
}
return msg
}
+
+// PruneUpgradeLogs removes expired upgrade logs from this instance's data directory.
+func (s *SystemUpgradeService) PruneUpgradeLogs(ctx context.Context, dataDir string, now time.Time) (int, error) {
+ retentionDays := s.settingsService.GetIntSetting(ctx, "upgradeLogRetentionDays", 3)
+ if retentionDays == 0 {
+ return 0, nil
+ }
+ if retentionDays < 0 || retentionDays > 3650 {
+ return 0, errors.New("upgrade log retention must be between 0 and 3650 days")
+ }
+ if err := ctx.Err(); err != nil {
+ return 0, err
+ }
+
+ root, err := os.OpenRoot(dataDir)
+ if errors.Is(err, os.ErrNotExist) {
+ return 0, nil
+ }
+ if err != nil {
+ return 0, errors.WrapIf(err, "open upgrade log directory")
+ }
+ defer root.Close()
+ entries, err := fs.ReadDir(root.FS(), ".")
+ if err != nil {
+ return 0, errors.WrapIf(err, "read upgrade log directory")
+ }
+
+ cutoff := now.Add(-time.Duration(retentionDays) * 24 * time.Hour)
+ removed := 0
+ var failures []error
+ for _, entry := range entries {
+ if err := ctx.Err(); err != nil {
+ return removed, stderrors.Join(append(failures, err)...)
+ }
+ if !upgradeLogNameInternal.MatchString(entry.Name()) {
+ continue
+ }
+ info, err := root.Lstat(entry.Name())
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ failures = append(failures, errors.WrapIff(err, "inspect upgrade log %s", entry.Name()))
+ continue
+ }
+ if !info.Mode().IsRegular() || !info.ModTime().Before(cutoff) {
+ continue
+ }
+ if err := root.Remove(entry.Name()); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ failures = append(failures, errors.WrapIff(err, "remove upgrade log %s", entry.Name()))
+ }
+ continue
+ }
+ removed++
+ }
+ return removed, stderrors.Join(failures...)
+}
diff --git a/backend/pkg/libarcane/internal_resource.go b/backend/pkg/libarcane/internal_resource.go
index 82f49e905..cf5fd8c1c 100644
--- a/backend/pkg/libarcane/internal_resource.go
+++ b/backend/pkg/libarcane/internal_resource.go
@@ -16,6 +16,8 @@ import (
)
const (
+ // UpgradeLogDirectory is mounted into the upgrader to persist its logs.
+ UpgradeLogDirectory = "/app/data"
// InternalResourceLabel marks containers used for Arcane utilities.
InternalResourceLabel = "com.getarcaneapp.internal.resource"
// HiddenResourceLabel hides containers from container lists and dashboard counts.
diff --git a/backend/pkg/scheduler/upgrade_log_cleanup_job.go b/backend/pkg/scheduler/upgrade_log_cleanup_job.go
new file mode 100644
index 000000000..68625169c
--- /dev/null
+++ b/backend/pkg/scheduler/upgrade_log_cleanup_job.go
@@ -0,0 +1,32 @@
+package scheduler
+
+import (
+ "context"
+ "log/slog"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/system"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+const UpgradeLogCleanupJobName = "upgrade-log-cleanup"
+
+// NewUpgradeLogCleanupJob creates the hourly local upgrade log cleanup job.
+func NewUpgradeLogCleanupJob(service *system.SystemUpgradeService) schedulertypes.Job {
+ return &schedulertypes.GenericJob{
+ JobName: UpgradeLogCleanupJobName,
+ ScheduleFn: func(context.Context) string { return "0 15 * * * *" },
+ RunFn: func(ctx context.Context) (schedulertypes.Outcome, error) {
+ removed, err := service.PruneUpgradeLogs(ctx, libarcane.UpgradeLogDirectory, time.Now())
+ if err != nil {
+ slog.ErrorContext(ctx, "Failed to clean up upgrade logs", "jobName", UpgradeLogCleanupJobName, "removed", removed, "error", err)
+ return schedulertypes.Outcome{}, err
+ }
+ if removed > 0 {
+ slog.InfoContext(ctx, "Removed expired upgrade logs", "jobName", UpgradeLogCleanupJobName, "removed", removed)
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
+ },
+ }
+}
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 02ea53ad7..32ed75839 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -2324,6 +2324,10 @@
"activity_cancel_already_finished": "Activity already finished",
"activity_settings_description": "Configure Activity Center history retention for each environment.",
"activity_settings_saved": "Activity settings saved",
+ "upgrade_logs_section_title": "Upgrade logs",
+ "upgrade_log_retention_days": "Upgrade log retention (days)",
+ "upgrade_log_retention_days_description": "Delete upgrade log files older than this many days. Defaults to 3 days.",
+ "upgrade_log_retention_days_help": "Cleanup runs hourly for this environment. Set 0 to keep logs indefinitely. Changes apply on the next cleanup run.",
"activity_history_section_title": "History Retention",
"activity_history_retention_days": "Retention Days",
"activity_history_retention_days_description": "Delete completed Activity Center entries older than this many days.",
diff --git a/frontend/src/lib/services/settings-service.ts b/frontend/src/lib/services/settings-service.ts
index 75d7deb62..6ce81f5cf 100644
--- a/frontend/src/lib/services/settings-service.ts
+++ b/frontend/src/lib/services/settings-service.ts
@@ -138,7 +138,7 @@ class SettingsService extends BaseAPIService {
}
if (value === 'true') return true;
if (value === 'false') return false;
- if (key === 'projectWorkspaceMaxFileSizeMb' || key === 'volumeWorkspaceMaxFileSizeMb') {
+ if (key === 'projectWorkspaceMaxFileSizeMb' || key === 'volumeWorkspaceMaxFileSizeMb' || key === 'upgradeLogRetentionDays') {
return Number(value);
}
return value;
diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.ts
index 47ad27daf..3c4efb55d 100644
--- a/frontend/src/lib/types/settings.ts
+++ b/frontend/src/lib/types/settings.ts
@@ -30,6 +30,7 @@ export type Settings = {
dockerClientRefreshInterval?: string;
environmentHealthInterval: number;
activityHistoryRetentionDays: number;
+ upgradeLogRetentionDays: number;
activityHistoryMaxEntries: number;
maxConcurrentActivities: number;
defaultDeployPullPolicy: 'missing' | 'always' | 'never';
diff --git a/frontend/src/routes/(app)/settings/activity/+page.svelte b/frontend/src/routes/(app)/settings/activity/+page.svelte
index c9259d550..bf9700fdb 100644
--- a/frontend/src/routes/(app)/settings/activity/+page.svelte
+++ b/frontend/src/routes/(app)/settings/activity/+page.svelte
@@ -13,6 +13,7 @@
const formSchema = z.object({
activityHistoryRetentionDays: z.coerce.number().int().min(0).max(3650),
+ upgradeLogRetentionDays: z.coerce.number().int().min(0).max(3650),
activityHistoryMaxEntries: z.coerce.number().int().min(0).max(100000),
maxConcurrentActivities: z.coerce.number().int().min(0).max(1000)
});
@@ -21,6 +22,7 @@
const settings = settingsStore.current || data.settings!;
return {
activityHistoryRetentionDays: settings.activityHistoryRetentionDays,
+ upgradeLogRetentionDays: settings.upgradeLogRetentionDays ?? 3,
activityHistoryMaxEntries: settings.activityHistoryMaxEntries,
maxConcurrentActivities: settings.maxConcurrentActivities
};
@@ -75,6 +77,19 @@
/>
</div>
</div>
+ <div class="space-y-4">
+ <h3 class="text-base font-semibold">{m.upgrade_logs_section_title()}</h3>
+ <div class="grid gap-5 sm:grid-cols-2 lg:grid-cols-3">
+ <TextInputWithLabel
+ bind:value={formInputs.upgradeLogRetentionDays.value}
+ error={formInputs.upgradeLogRetentionDays.error}
+ label={m.upgrade_log_retention_days()}
+ description={m.upgrade_log_retention_days_description()}
+ helpText={m.upgrade_log_retention_days_help()}
+ type="number"
+ />
+ </div>
+ </div>
</fieldset>
{/snippet}
</SettingsPageLayout>
diff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.go
index fec508f40..6e0569af8 100644
--- a/types/meta/job_metadata.go
+++ b/types/meta/job_metadata.go
@@ -59,6 +59,16 @@ var jobMetadataRegistry = map[string]JobMetadata{
CanRunManually: true,
Prerequisites: []JobPrerequisiteMetadata{},
},
+ "upgrade-log-cleanup": {
+ ID: "upgrade-log-cleanup",
+ Name: "Upgrade Log Cleanup",
+ Description: "Removes upgrade logs older than the configured retention period",
+ Category: "maintenance",
+ ManagerOnly: false,
+ IsContinuous: false,
+ CanRunManually: true,
+ Prerequisites: []JobPrerequisiteMetadata{},
+ },
"expired-sessions-cleanup": {
ID: "expired-sessions-cleanup",
Name: "Expired Sessions Cleanup",
diff --git a/types/settings/settings.go b/types/settings/settings.go
index e6173fc8d..b937d8e5a 100644
--- a/types/settings/settings.go
+++ b/types/settings/settings.go
@@ -104,6 +104,11 @@ type Update struct {
// Required: false
ActivityHistoryRetentionDays *string `json:"activityHistoryRetentionDays,omitzero"`
+ // UpgradeLogRetentionDays is the upgrade log retention in days. Zero disables cleanup.
+ //
+ // Required: false
+ UpgradeLogRetentionDays *string `json:"upgradeLogRetentionDays,omitzero"`
+
// ActivityHistoryMaxEntries is the maximum completed Activity Center entries to retain per environment.
//
// Required: false