Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

19.9 KB
Raw Blame
<?php

namespace App\Actions\Server;

use App\Enums\ApplicationDeploymentStatus;
use App\Models\ApplicationDeploymentQueue;
use App\Models\Server;
use App\Support\Actions\UniqueUntilProcessingJobDecorator;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Lorisleiva\Actions\Concerns\AsAction;
use Lorisleiva\Actions\Decorators\JobDecorator;
use Lorisleiva\Actions\Decorators\UniqueJobDecorator;
use RuntimeException;

class CleanupDocker implements ShouldBeUnique
{
    use AsAction;

    /**
     * Timeout of a queued cleanup, shared with DockerCleanupJob.
     */
    public const JOB_TIMEOUT = 600;

    /**
     * Time budget for all remote commands of one cleanup. It stays below JOB_TIMEOUT, so a
     * slow server makes the cleanup fail with an error instead of the worker killing the job.
     * A remote Docker prune that already started still finishes on the server.
     */
    public const REMOTE_COMMANDS_DEADLINE = 540;

    /**
     * Delay of a queued cleanup. Stop requests for one server in this window (for example a
     * bulk stop through the API) become one cleanup that runs after the batch.
     */
    public const QUEUED_DELAY = 60;

    /**
     * A resource stop does not queue a cleanup when a cleanup finished on the server in this
     * window. The scheduled cleanup still runs.
     */
    public const STOP_CLEANUP_COOLDOWN = 3600;

    /**
     * Cleanup log command of the entry that records a skipped build cache prune.
     */
    public const BUILD_CACHE_KEPT_LOG_COMMAND = 'docker builder prune, docker buildx prune';

    public int $jobTries = 1;

    /**
     * Bounds how long a lost queued run (for example a killed worker) blocks new cleanups.
     */
    public int $jobUniqueFor = self::QUEUED_DELAY + 240;

    public int $jobTimeout = self::JOB_TIMEOUT;

    private int $deadline = 0;

    /**
     * Overlap lock key shared by DockerCleanupJob and queued CleanupDocker runs.
     */
    public static function overlapLockKey(Server $server): string
    {
        return 'docker-cleanup-'.$server->uuid;
    }

    /**
     * Lock lifetime: longer than the job timeout, so a new run cannot start while a timed-out
     * run is still finishing.
     */
    public static function overlapLockExpiresAfter(): int
    {
        return self::JOB_TIMEOUT + 60;
    }

    /**
     * At most one queued cleanup per server. The unique lock is released when a worker starts
     * the job, so a stop during a running cleanup queues one follow-up cleanup.
     */
    public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator
    {
        return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments);
    }

    public static function lastRunCacheKey(Server $server): string
    {
        return 'docker-cleanup-last-run-'.$server->uuid;
    }

    /**
     * The deployment that is queued or running on the server (as target or build server), if any.
     * Pruning the build caches fails a build that uses them, so the cleanup keeps them while such a
     * deployment exists. A deployment that wrote no log for longer than the deployment timeout is
     * stuck and does not count.
     */
    public static function activeDeployment(Server $server): ?ApplicationDeploymentQueue
    {
        return ApplicationDeploymentQueue::query()
            ->where(fn ($query) => $query->where('server_id', $server->id)->orWhere('build_server_id', $server->id))
            ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value])
            ->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600))
            ->first(['id', 'deployment_uuid']);
    }

    /**
     * The reason the build cache was kept, read from a cleanup log, if it was kept.
     */
    public static function buildCacheKeptReason(array $cleanupLog): ?string
    {
        return collect($cleanupLog)->firstWhere('command', self::BUILD_CACHE_KEPT_LOG_COMMAND)['output'] ?? null;
    }

    /**
     * Cleanup after a resource stop: skipped when a cleanup ran recently, and the queued run
     * only cleans when the disk usage is at or above the server cleanup threshold.
     */
    public static function dispatchAfterStop(Server $server): void
    {
        if (Cache::has(self::lastRunCacheKey($server))) {
            return;
        }

        static::dispatch($server, false, false, true);
    }

    public function getJobUniqueId(Server $server): string
    {
        return $server->uuid;
    }

    public function configureJob(JobDecorator $job): void
    {
        $job->onQueue(maintenance_queue())->delay(self::QUEUED_DELAY);
    }

    /**
     * Only applies when dispatched as a job. ::run() inside DockerCleanupJob already holds
     * the lock and does not go through this middleware.
     */
    public function getJobMiddleware(Server $server): array
    {
        return [
            (new WithoutOverlapping(self::overlapLockKey($server)))
                ->shared()
                ->expireAfter(self::overlapLockExpiresAfter())
                ->dontRelease(),
        ];
    }

    /**
     * @param  bool  $pruneBuildCacheDuringDeployment  Also prune the build caches while a deployment is active
     *                                                 (a manual cleanup). Otherwise they stay until the next run.
     */
    public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false, bool $onlyAboveThreshold = false, bool $pruneBuildCacheDuringDeployment = false)
    {
        $this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE;

        if ($onlyAboveThreshold) {
            $diskUsage = $server->getDiskUsage();
            if (is_numeric($diskUsage) && (int) $diskUsage < $server->settings->docker_cleanup_threshold) {
                return [];
            }
        }
        $helperImageVersion = getHelperVersion();
        $helperImageWithVersion = coolifyHelperImage().":$helperImageVersion";

        $cleanupLog = [];

        // Get all application image repositories to exclude from prune
        $applications = $server->applications();
        $applicationImageRepos = collect($applications)->map(function ($app) {
            return $app->docker_registry_image_name ?? $app->uuid;
        })->unique()->values();

        // Clean up old application images while preserving N most recent for rollback
        $applicationCleanupLog = $this->cleanupApplicationImages($server, $applications);
        $cleanupLog = array_merge($cleanupLog, $applicationCleanupLog);

        // Build image prune command that excludes application images and current Coolify infrastructure images
        // This ensures we clean up non-Coolify images while preserving rollback images and current helper image
        // Note: Only the current version is protected; old versions will be cleaned up by explicit commands below
        // We pass the version strings so all registry variants are protected (ghcr.io, docker.io, no prefix)
        $imagePruneCmd = $this->buildImagePruneCommand(
            $applicationImageRepos,
            $helperImageVersion
        );

        $commands = [
            $this->buildContainerPruneCommand(),
            $imagePruneCmd,
        ];

        $activeDeployment = $pruneBuildCacheDuringDeployment ? null : self::activeDeployment($server);
        if (! $activeDeployment) {
            $commands[] = 'docker builder prune -af';
            // The prune fails on a stopped builder, so start it first.
            $commands[] = railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, railpackBuilderBootstrapScript().'; docker buildx prune --builder coolify-railpack -af');
        } else {
            $cleanupLog[] = [
                'command' => self::BUILD_CACHE_KEPT_LOG_COMMAND,
                'output' => 'Skipped: the build cache was not pruned because deployment '.$activeDeployment->deployment_uuid.' is queued or running on this server.',
            ];
        }

        $commands[] = $this->buildCoolifyImageCleanupCommand($helperImageVersion);

        if ($deleteUnusedVolumes) {
            $commands[] = 'docker volume prune -af';
        }

        if ($deleteUnusedNetworks) {
            $commands[] = 'docker network prune -f';
        }

        foreach ($commands as $command) {
            $commandOutput = $this->runRemoteCommand($command, $server);
            if ($commandOutput !== null) {
                $cleanupLog[] = [
                    'command' => $command,
                    'output' => remove_iip($commandOutput),
                ];
            }
        }

        Cache::put(self::lastRunCacheKey($server), true, self::STOP_CLEANUP_COOLDOWN);

        return $cleanupLog;
    }

    /**
     * Run one remote command with the time left before the cleanup deadline as its local
     * SSH timeout. Errors and timeouts of a single command are ignored, as before; once the
     * deadline has passed the cleanup stops and fails.
     */
    private function runRemoteCommand(string $command, Server $server): ?string
    {
        $remaining = $this->deadline - now()->getTimestamp();
        if ($remaining < 1) {
            throw new RuntimeException('Docker cleanup did not finish within '.self::REMOTE_COMMANDS_DEADLINE.' seconds. The remaining cleanup commands were not run.');
        }

        return instant_remote_process([$command], $server, false, timeout: $remaining);
    }

    /**
     * Build a command that removes stopped Coolify containers, except the proxy and the
     * containers of databases, applications and services.
     *
     * docker container prune only excludes a container that matches all of its label!= filters
     * together, so we list the stopped containers and filter them by label in the shell.
     */
    private function buildContainerPruneCommand(): string
    {
        return "docker ps -a --filter status=created --filter status=exited --filter status=dead --filter label=coolify.managed=true --format '{{.ID}} coolify.proxy={{.Label \"coolify.proxy\"}} coolify.type={{.Label \"coolify.type\"}}' | ".
            "grep -v -E 'coolify\.proxy=true |coolify\.type=(database|application|service)$' | ".
            "awk '{print $1}' | xargs -r docker rm";
    }

    /**
     * Build a command that removes unused images of the Coolify helper, realtime and sentinel,
     * from any registry. They have the coolify.managed=true label, so the generic image prune
     * skips them. The current helper version stays, and docker rmi (without -f) keeps images
     * that a container uses, for example the running sentinel.
     */
    private function buildCoolifyImageCleanupCommand(string $helperImageVersion): string
    {
        $escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion);

        return "docker images --format '{{.Repository}}:{{.Tag}}' | ".
            "grep -E '(^|/)coollabsio/(coolify-helper|coolify-realtime|sentinel):' | ".
            "grep -v -E ':<none>$|(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$' | ".
            'xargs -r docker rmi 2>/dev/null || true';
    }

    /**
     * Build a docker image prune command that excludes application image repositories.
     *
     * Since docker image prune doesn't support excluding by repository name directly,
     * we use a shell script approach to delete unused images while preserving application images.
     */
    private function buildImagePruneCommand(
        $applicationImageRepos,
        string $helperImageVersion
    ): string {
        // Step 1: Always prune dangling images (untagged)
        $commands = ['docker image prune -f'];

        // Build grep pattern to exclude application image repositories (matches repo:tag and repo_service:tag)
        $appExcludePatterns = $applicationImageRepos->map(function ($repo) {
            // Escape special characters for grep extended regex (ERE)
            // ERE special chars: . \ + * ? [ ^ ] $ ( ) { } |
            return preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $repo);
        })->implode('|');

        // Build grep pattern to exclude Coolify infrastructure images (current version only)
        // This pattern matches the image name regardless of registry prefix:
        // - ghcr.io/coollabsio/coolify-helper:1.0.12
        // - docker.io/coollabsio/coolify-helper:1.0.12
        // - coollabsio/coolify-helper:1.0.12
        // Pattern: (^|/)coollabsio/coolify-helper:VERSION$
        $escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion);
        $infraExcludePattern = "(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$";

        // Delete unused images that:
        // - Are not application images (don't match app repos)
        // - Are not current Coolify helper image (any registry)
        // - Don't have coolify.managed=true label
        // Images in use by containers will fail silently with docker rmi
        // Pattern matches both uuid:tag and uuid_servicename:tag (Docker Compose with build)
        // Images pulled by digest have no tag and are not dangling, so they are listed as repo@digest
        $grepCommands = "grep -v '<none>'";

        // Add application repo exclusion if there are applications
        if ($applicationImageRepos->isNotEmpty()) {
            // The repository names come from application data, so quote the pattern for the shell.
            $grepCommands .= ' | grep -v -E '.escapeshellarg("^({$appExcludePatterns})[_:@].+");
        }

        // Add infrastructure image exclusion (matches any registry prefix)
        $grepCommands .= ' | grep -v -E '.escapeshellarg($infraExcludePattern);

        $commands[] = "docker images --digests --format '{{if eq .Tag \"<none>\"}}{{.Repository}}@{{.Digest}}{{else}}{{.Repository}}:{{.Tag}}{{end}}' | ".
            $grepCommands.' | '.
            "xargs -r -I {} sh -c 'docker inspect --format \"{{index .Config.Labels \\\"coolify.managed\\\"}}\" \"{}\" 2>/dev/null | grep -q true || docker rmi \"{}\" 2>/dev/null' || true";

        return implode(' && ', $commands);
    }

    /**
     * Remove old application images while the N most recent images of each application stay
     * for rollback. Uses a fixed number of remote commands, independent of the number of
     * applications on the server.
     */
    private function cleanupApplicationImages(Server $server, $applications = null): array
    {
        $cleanupLog = [];

        if ($applications === null) {
            $applications = $server->applications();
        }
        if ($applications->isEmpty()) {
            return $cleanupLog;
        }
        $applications->loadMissing('settings');

        $disableRetention = $server->settings->disable_application_image_retention ?? false;

        // Images that a container uses, running or stopped
        $containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? ''))
            ->filter()
            ->map(fn (string $line) => array_pad(explode('#', $line, 2), 2, '')[1])
            ->unique();

        $serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? ''))
            ->filter()
            ->map(function (string $line) {
                [$repository, $tag, $createdAt] = array_pad(explode('#', $line, 3), 3, '');

                return [
                    'repository' => $repository,
                    'tag' => $tag,
                    'created_at' => $createdAt,
                    'image_ref' => "{$repository}:{$tag}",
                ];
            })
            ->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== '<none>');

        // A deployment starts the containers of its new images only after the build or pull,
        // so the images of an application with a queued or running deployment all stay.
        $deployingApplicationIds = ApplicationDeploymentQueue::query()
            ->whereIn('application_id', $applications->map(fn ($application) => (string) $application->id))
            ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value])
            ->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600))
            ->pluck('application_id');

        $imagesToDelete = collect();

        foreach ($applications as $application) {
            if ($deployingApplicationIds->contains((string) $application->id)) {
                continue;
            }

            $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2);
            $imageRepository = $application->docker_registry_image_name ?? $application->uuid;

            // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like
            // the docker images --filter reference='<repository>*' filter. Each repository (the
            // application, or one Docker Compose service) has its own retention.
            $repositories = $serverImages
                ->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository)
                    && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/'))
                ->groupBy('repository');

            foreach ($repositories as $images) {
                // Tags that a container uses
                $currentTags = $images->filter(fn ($image) => $containerImages->contains($image['image_ref']))->pluck('tag');

                // Separate images into categories
                // PR images (pr-*) are always deleted
                // Build images (*-build) are cleaned up to match retained regular images
                $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-'));
                $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build'));
                $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build'));

                // Filter out the images in use from regular images and sort by creation date
                $sortedRegularImages = $regularImages
                    ->reject(fn ($image) => $currentTags->contains($image['tag']))
                    ->sortByDesc('created_at')
                    ->values();

                // Clean up build images (-build suffix) that don't correspond to retained regular images
                // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers.
                $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag')->concat($currentTags);

                $imagesToDelete = $imagesToDelete
                    ->concat($prImages)
                    ->concat($sortedRegularImages->skip($imagesToKeep))
                    ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag']))));
            }
        }

        // Images in use fail silently; docker rmi continues with the next image
        foreach ($this->imageRemovalCommands($imagesToDelete->pluck('image_ref')->unique()) as $deleteCommand) {
            $deleteOutput = $this->runRemoteCommand($deleteCommand, $server);
            $cleanupLog[] = [
                'command' => $deleteCommand,
                'output' => remove_iip($deleteOutput ?? 'Images removed or were in use'),
            ];
        }

        return $cleanupLog;
    }

    /**
     * @param  Collection<int, string>  $imageRefs
     * @return Collection<int, string>
     */
    private function imageRemovalCommands(Collection $imageRefs): Collection
    {
        return $imageRefs
            ->chunk(100)
            ->map(fn (Collection $chunk) => 'docker rmi '.$chunk->map(fn (string $ref) => escapeshellarg($ref))->implode(' ').' 2>/dev/null || true')
            ->values();
    }
}