Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(docker-cleanup): run one bounded cleanup per server on the maintenance queue

Andras Bacsai7956be8833ab
diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php
index 9f58fd4f88..cde75e30de 100644
--- a/app/Actions/Server/CleanupDocker.php
+++ b/app/Actions/Server/CleanupDocker.php
@@ -3,16 +3,101 @@
namespace App\Actions\Server;
use App\Models\Server;
+use App\Support\Actions\UniqueUntilProcessingJobDecorator;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Support\Collection;
use Lorisleiva\Actions\Concerns\AsAction;
+use Lorisleiva\Actions\Decorators\JobDecorator;
+use Lorisleiva\Actions\Decorators\UniqueJobDecorator;
+use RuntimeException;
-class CleanupDocker
+class CleanupDocker implements ShouldBeUnique
{
use AsAction;
- public string $jobQueue = 'high';
+ /**
+ * Timeout of a queued cleanup, shared with DockerCleanupJob.
+ */
+ public const JOB_TIMEOUT = 600;
+
+ /**
+ * Time budget for all remote commands of one cleanup. It stays below JOB_TIMEOUT, so a
+ * slow server makes the cleanup fail with an error instead of the worker killing the job.
+ * A remote Docker prune that already started still finishes on the server.
+ */
+ public const REMOTE_COMMANDS_DEADLINE = 540;
+
+ /**
+ * Delay of a queued cleanup. Stop requests for one server in this window (for example a
+ * bulk stop through the API) become one cleanup that runs after the batch.
+ */
+ public const QUEUED_DELAY = 60;
+
+ public int $jobTries = 1;
+
+ /**
+ * Bounds how long a lost queued run (for example a killed worker) blocks new cleanups.
+ */
+ public int $jobUniqueFor = self::QUEUED_DELAY + 240;
+
+ public int $jobTimeout = self::JOB_TIMEOUT;
+
+ private int $deadline = 0;
+
+ /**
+ * Overlap lock key shared by DockerCleanupJob and queued CleanupDocker runs.
+ */
+ public static function overlapLockKey(Server $server): string
+ {
+ return 'docker-cleanup-'.$server->uuid;
+ }
+
+ /**
+ * Lock lifetime: longer than the job timeout, so a new run cannot start while a timed-out
+ * run is still finishing.
+ */
+ public static function overlapLockExpiresAfter(): int
+ {
+ return self::JOB_TIMEOUT + 60;
+ }
+
+ /**
+ * At most one queued cleanup per server. The unique lock is released when a worker starts
+ * the job, so a stop during a running cleanup queues one follow-up cleanup.
+ */
+ public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator
+ {
+ return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments);
+ }
+
+ public function getJobUniqueId(Server $server): string
+ {
+ return $server->uuid;
+ }
+
+ public function configureJob(JobDecorator $job): void
+ {
+ $job->onQueue(maintenance_queue())->delay(self::QUEUED_DELAY);
+ }
+
+ /**
+ * Only applies when dispatched as a job. ::run() inside DockerCleanupJob already holds
+ * the lock and does not go through this middleware.
+ */
+ public function getJobMiddleware(Server $server): array
+ {
+ return [
+ (new WithoutOverlapping(self::overlapLockKey($server)))
+ ->shared()
+ ->expireAfter(self::overlapLockExpiresAfter())
+ ->dontRelease(),
+ ];
+ }
public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false)
{
+ $this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE;
$helperImageVersion = getHelperVersion();
$helperImage = coolifyHelperImage();
$helperImageWithVersion = "$helperImage:$helperImageVersion";
@@ -61,7 +146,7 @@ class CleanupDocker
}
foreach ($commands as $command) {
- $commandOutput = instant_remote_process([$command], $server, false);
+ $commandOutput = $this->runRemoteCommand($command, $server);
if ($commandOutput !== null) {
$cleanupLog[] = [
'command' => $command,
@@ -73,6 +158,21 @@ class CleanupDocker
return $cleanupLog;
}
+ /**
+ * Run one remote command with the time left before the cleanup deadline as its local
+ * SSH timeout. Errors and timeouts of a single command are ignored, as before; once the
+ * deadline has passed the cleanup stops and fails.
+ */
+ private function runRemoteCommand(string $command, Server $server): ?string
+ {
+ $remaining = $this->deadline - now()->getTimestamp();
+ if ($remaining < 1) {
+ throw new RuntimeException('Docker cleanup did not finish within '.self::REMOTE_COMMANDS_DEADLINE.' seconds. The remaining cleanup commands were not run.');
+ }
+
+ return instant_remote_process([$command], $server, false, timeout: $remaining);
+ }
+
/**
* Build a docker image prune command that excludes application image repositories.
*
@@ -125,6 +225,11 @@ class CleanupDocker
return implode(' && ', $commands);
}
+ /**
+ * Remove old application images while the N most recent images of each application stay
+ * for rollback. Uses a fixed number of remote commands, independent of the number of
+ * applications on the server.
+ */
private function cleanupApplicationImages(Server $server, $applications = null): array
{
$cleanupLog = [];
@@ -132,43 +237,54 @@ class CleanupDocker
if ($applications === null) {
$applications = $server->applications();
}
+ if ($applications->isEmpty()) {
+ return $cleanupLog;
+ }
+ $applications->loadMissing('settings');
$disableRetention = $server->settings->disable_application_image_retention ?? false;
+ // Image of each container, keyed by container name
+ $containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? ''))
+ ->filter()
+ ->mapWithKeys(function (string $line) {
+ [$name, $image] = array_pad(explode('#', $line, 2), 2, '');
+
+ return [$name => $image];
+ });
+
+ $serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? ''))
+ ->filter()
+ ->map(function (string $line) {
+ [$repository, $tag, $createdAt] = array_pad(explode('#', $line, 3), 3, '');
+
+ return [
+ 'repository' => $repository,
+ 'tag' => $tag,
+ 'created_at' => $createdAt,
+ 'image_ref' => "{$repository}:{$tag}",
+ ];
+ })
+ ->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== '<none>');
+
+ $imagesToDelete = collect();
+
foreach ($applications as $application) {
$imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2);
$imageRepository = $application->docker_registry_image_name ?? $application->uuid;
- // Get the currently running image tag
- $currentTagCommand = "docker inspect --format='{{.Config.Image}}' {$application->uuid} 2>/dev/null | grep -oP '(?<=:)[^:]+$' || true";
- $currentTag = instant_remote_process([$currentTagCommand], $server, false);
- $currentTag = trim($currentTag ?? '');
+ // Tag of the image of the running container (named after the application uuid)
+ $currentTag = preg_match('/:([^:]+)$/', $containerImages->get($application->uuid, ''), $matches) ? $matches[1] : '';
- // List all images for this application with their creation timestamps
- // Use wildcard to match both uuid:tag and uuid_servicename:tag (Docker Compose with build)
- $listCommand = "docker images --format '{{.Repository}}:{{.Tag}}#{{.CreatedAt}}' --filter reference='{$imageRepository}*' 2>/dev/null || true";
- $output = instant_remote_process([$listCommand], $server, false);
+ // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like
+ // the docker images --filter reference='<repository>*' filter
+ $images = $serverImages->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository)
+ && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/'));
- if (empty($output)) {
+ if ($images->isEmpty()) {
continue;
}
- $images = collect(explode("\n", trim($output)))
- ->filter()
- ->map(function ($line) {
- $parts = explode('#', $line);
- $imageRef = $parts[0] ?? '';
- $tagParts = explode(':', $imageRef);
-
- return [
- 'repository' => $tagParts[0] ?? '',
- 'tag' => $tagParts[1] ?? '',
- 'created_at' => $parts[1] ?? '',
- 'image_ref' => $imageRef,
- ];
- })
- ->filter(fn ($image) => ! empty($image['tag']));
-
// Separate images into categories
// PR images (pr-*) are always deleted
// Build images (*-build) are cleaned up to match retained regular images
@@ -176,55 +292,47 @@ class CleanupDocker
$buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build'));
$regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build'));
- // Always delete all PR images
- foreach ($prImages as $image) {
- $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
- $deleteOutput = instant_remote_process([$deleteCommand], $server, false);
- $cleanupLog[] = [
- 'command' => $deleteCommand,
- 'output' => $deleteOutput ?? 'PR image removed or was in use',
- ];
- }
-
// Filter out current running image from regular images and sort by creation date
$sortedRegularImages = $regularImages
->filter(fn ($image) => $image['tag'] !== $currentTag)
->sortByDesc('created_at')
->values();
- // Keep only N images (imagesToKeep), delete the rest
- $imagesToDelete = $sortedRegularImages->skip($imagesToKeep);
-
- foreach ($imagesToDelete as $image) {
- $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
- $deleteOutput = instant_remote_process([$deleteCommand], $server, false);
- $cleanupLog[] = [
- 'command' => $deleteCommand,
- 'output' => $deleteOutput ?? 'Image removed or was in use',
- ];
- }
-
// Clean up build images (-build suffix) that don't correspond to retained regular images
// Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers.
// If a build is in progress, docker rmi will fail silently since the image is in use.
$keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag');
- if (! empty($currentTag)) {
- $keptTags = $keptTags->push($currentTag);
+ if ($currentTag !== '') {
+ $keptTags->push($currentTag);
}
- foreach ($buildImages as $image) {
- $baseTag = preg_replace('/-build$/', '', $image['tag']);
- if (! $keptTags->contains($baseTag)) {
- $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
- $deleteOutput = instant_remote_process([$deleteCommand], $server, false);
- $cleanupLog[] = [
- 'command' => $deleteCommand,
- 'output' => $deleteOutput ?? 'Build image removed or was in use',
- ];
- }
- }
+ $imagesToDelete = $imagesToDelete
+ ->concat($prImages)
+ ->concat($sortedRegularImages->skip($imagesToKeep))
+ ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag']))));
+ }
+
+ // Images in use fail silently; docker rmi continues with the next image
+ foreach ($this->imageRemovalCommands($imagesToDelete->pluck('image_ref')->unique()) as $deleteCommand) {
+ $deleteOutput = $this->runRemoteCommand($deleteCommand, $server);
+ $cleanupLog[] = [
+ 'command' => $deleteCommand,
+ 'output' => $deleteOutput ?? 'Images removed or were in use',
+ ];
}
return $cleanupLog;
}
+
+ /**
+ * @param Collection<int, string> $imageRefs
+ * @return Collection<int, string>
+ */
+ private function imageRemovalCommands(Collection $imageRefs): Collection
+ {
+ return $imageRefs
+ ->chunk(100)
+ ->map(fn (Collection $chunk) => 'docker rmi '.$chunk->map(fn (string $ref) => escapeshellarg($ref))->implode(' ').' 2>/dev/null || true')
+ ->values();
+ }
}
diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php
index 4c744a6e64..03a9207827 100644
--- a/app/Jobs/DockerCleanupJob.php
+++ b/app/Jobs/DockerCleanupJob.php
@@ -15,14 +15,13 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
- public $timeout = 600;
+ public $timeout = CleanupDocker::JOB_TIMEOUT;
public $tries = 1;
@@ -30,9 +29,20 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
public ?DockerCleanupExecution $execution_log = null;
+ /**
+ * Shares the per-server lock with queued CleanupDocker runs, so two cleanups never run on
+ * one server at once. A scheduled run that finds the lock held marks its occurrence
+ * skipped (not missed); a manual run is dropped as before. The lock outlives the job
+ * timeout, so a new run cannot start while a timed-out run is still finishing.
+ */
public function middleware(): array
{
- return [(new WithoutOverlapping('docker-cleanup-'.$this->server->uuid))->expireAfter(600)->dontRelease()];
+ return [
+ ScheduledJobDeliveryService::withoutOverlapping(CleanupDocker::overlapLockKey($this->server), $this->occurrenceUuid)
+ ->shared()
+ ->expireAfter(CleanupDocker::overlapLockExpiresAfter())
+ ->dontRelease(),
+ ];
}
public function __construct(
@@ -42,7 +52,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
public bool $deleteUnusedNetworks = false,
public ?string $occurrenceUuid = null,
) {
- $this->onQueue('high');
+ $this->onQueue(maintenance_queue());
}
public function handle(): void
diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php
index 081b49a520..eeedb60559 100644
--- a/tests/Feature/DockerCleanupJobTest.php
+++ b/tests/Feature/DockerCleanupJobTest.php
@@ -1,13 +1,76 @@
<?php
+use App\Actions\Server\CleanupDocker;
use App\Jobs\DockerCleanupJob;
+use App\Models\Application;
use App\Models\DockerCleanupExecution;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\ScheduledJobDelivery;
use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
use App\Models\User;
+use App\Notifications\Server\DockerCleanupFailed;
+use App\Services\ScheduledJobDeliveryService;
+use App\Support\Actions\UniqueUntilProcessingJobDecorator;
use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use Lorisleiva\Actions\Decorators\JobDecorator;
uses(RefreshDatabase::class);
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+});
+
+afterEach(function () {
+ Server::flushIdentityMap();
+ Carbon::setTestNow();
+});
+
+function dockerCleanupReachableServer(): Server
+{
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+
+ return $server->fresh();
+}
+
+/**
+ * @return Collection<int, string> The local commands (SSH invocations) that ran.
+ */
+function dockerCleanupFakeProcesses(?Closure $onRun = null): Collection
+{
+ $commands = collect();
+ Process::fake(function ($process) use ($commands, $onRun) {
+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+ $commands->push($command);
+ if ($onRun) {
+ $onRun($command);
+ }
+
+ return Process::result(output: '');
+ });
+
+ return $commands;
+}
+
it('persists the server id when creating an execution record', function () {
$user = User::factory()->create();
$team = $user->teams()->first();
@@ -86,3 +149,204 @@ it('finishes the latest running execution when the job fails after a timeout', f
->and($olderExecution->refresh()->status)->toBe('running')
->and($olderExecution->finished_at)->toBeNull();
});
+
+it('skips a scheduled cleanup occurrence while another cleanup holds the server lock, so it is not reported missed later', function () {
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
+ Notification::fake();
+ $server = dockerCleanupReachableServer();
+ $occurrence = ScheduledJobDelivery::create([
+ 'schedule_key' => "docker-cleanup:{$server->id}",
+ 'scheduled_for' => now()->startOfMinute(),
+ 'job_type' => 'docker-cleanup',
+ 'resource_id' => $server->id,
+ 'status' => 'enqueued',
+ 'enqueued_at' => now(),
+ ]);
+ $job = new DockerCleanupJob($server, false, false, false, $occurrence->uuid);
+ $runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
+ $runningCleanupLock->get();
+ $commands = dockerCleanupFakeProcesses();
+
+ dispatch_sync($job);
+ $runningCleanupLock->release();
+
+ expect($occurrence->fresh()->status)->toBe('skipped')
+ ->and(DockerCleanupExecution::query()->exists())->toBeFalse()
+ ->and($commands)->toBeEmpty();
+
+ Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1));
+ Queue::fake();
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+
+ Queue::assertNothingPushed();
+ expect(DockerCleanupExecution::query()->exists())->toBeFalse();
+ Notification::assertNotSentTo($server->team, DockerCleanupFailed::class);
+});
+
+it('drops a manual cleanup while another cleanup holds the server lock without touching occurrences', function () {
+ $server = dockerCleanupReachableServer();
+ $job = new DockerCleanupJob($server, true);
+ $runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
+ $runningCleanupLock->get();
+ $commands = dockerCleanupFakeProcesses();
+
+ dispatch_sync($job);
+ $runningCleanupLock->release();
+
+ expect(DockerCleanupExecution::query()->exists())->toBeFalse()
+ ->and(ScheduledJobDelivery::query()->exists())->toBeFalse()
+ ->and($commands)->toBeEmpty();
+});
+
+it('shares one server lock between scheduled cleanups and queued CleanupDocker runs that outlives the job timeout', function () {
+ $server = dockerCleanupReachableServer();
+ $cleanupJob = new DockerCleanupJob($server);
+ $actionJob = CleanupDocker::makeJob($server, false, false);
+
+ $cleanupLock = $cleanupJob->middleware()[0];
+ $actionLock = $actionJob->middleware()[0];
+
+ expect($cleanupLock->getLockKey($cleanupJob))
+ ->toBe('laravel-queue-overlap:docker-cleanup-'.$server->uuid)
+ ->toBe($actionLock->getLockKey($actionJob))
+ ->and($cleanupLock->expiresAfter)->toBeGreaterThan($cleanupJob->timeout)
+ ->and($actionLock->expiresAfter)->toBeGreaterThan($actionJob->timeout)
+ ->and($cleanupLock->releaseAfter)->toBeNull()
+ ->and($actionLock->releaseAfter)->toBeNull()
+ ->and($actionJob->timeout)->toBe(CleanupDocker::JOB_TIMEOUT)
+ ->and($actionJob->tries)->toBe(1);
+});
+
+it('drops a queued CleanupDocker run while a scheduled cleanup holds the server lock', function () {
+ $server = dockerCleanupReachableServer();
+ $actionJob = CleanupDocker::makeJob($server, false, false);
+ $runningCleanupLock = Cache::lock('laravel-queue-overlap:docker-cleanup-'.$server->uuid, 3600);
+ $runningCleanupLock->get();
+ $commands = dockerCleanupFakeProcesses();
+
+ dispatch_sync($actionJob);
+ $runningCleanupLock->release();
+
+ expect($commands)->toBeEmpty();
+});
+
+it('routes docker cleanups to the maintenance queue on cloud and to high on self-hosted', function (bool $selfHosted, string $queue) {
+ config(['constants.coolify.self_hosted' => $selfHosted]);
+ Queue::fake();
+ $server = dockerCleanupReachableServer();
+
+ DockerCleanupJob::dispatch($server, true);
+ CleanupDocker::dispatch($server, false, false);
+
+ expect(maintenance_queue())->toBe($queue);
+ Queue::assertPushedOn($queue, DockerCleanupJob::class);
+ Queue::assertPushedOn($queue, UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->decorates(CleanupDocker::class));
+})->with([
+ 'cloud' => [false, 'maintenance'],
+ 'self-hosted' => [true, 'high'],
+]);
+
+it('gives remote cleanup commands a local timeout and fails once the cleanup deadline has passed', function () {
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
+ $server = dockerCleanupReachableServer();
+ $commands = dockerCleanupFakeProcesses(function (string $command) {
+ if (str_contains($command, 'docker ')) {
+ Carbon::setTestNow(now()->addSeconds(300));
+ }
+ });
+
+ expect(fn () => CleanupDocker::run($server))
+ ->toThrow(RuntimeException::class, 'Docker cleanup did not finish within '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' seconds');
+
+ $sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker container prune') || str_contains($command, 'docker image prune'))->values();
+
+ expect(CleanupDocker::REMOTE_COMMANDS_DEADLINE)->toBeLessThan(CleanupDocker::JOB_TIMEOUT)
+ ->and($sshCommands)->toHaveCount(2)
+ ->and($sshCommands[0])->toContain('timeout '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' ssh')
+ ->and($sshCommands[1])->toContain('timeout '.(CleanupDocker::REMOTE_COMMANDS_DEADLINE - 300).' ssh')
+ ->and($commands->contains(fn (string $command) => str_contains($command, 'docker builder prune')))->toBeFalse();
+});
+
+it('coalesces queued cleanups per server and delays them until a bulk stop is over', function () {
+ Queue::fake();
+ $first = dockerCleanupReachableServer();
+ $second = dockerCleanupReachableServer();
+
+ foreach (range(1, 30) as $stop) {
+ CleanupDocker::dispatch($first, false, false);
+ }
+ CleanupDocker::dispatch($second, false, false);
+
+ Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2);
+ Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->delay === CleanupDocker::QUEUED_DELAY);
+});
+
+it('accepts a new queued cleanup once the waiting one is lost', function () {
+ Queue::fake();
+ $server = dockerCleanupReachableServer();
+
+ CleanupDocker::dispatch($server, false, false);
+ $this->travel(CleanupDocker::QUEUED_DELAY + 241)->seconds();
+ CleanupDocker::dispatch($server, false, false);
+
+ Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2);
+});
+
+it('scans application images with a fixed number of remote commands and keeps the retention rules', function () {
+ $server = dockerCleanupReachableServer();
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $server->team_id]);
+ $environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]);
+ $applications = collect(range(1, 5))->map(fn () => Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]));
+ $app = $applications->first();
+ $app->settings()->update(['docker_images_to_keep' => 1]);
+
+ $containers = "{$app->uuid}#{$app->uuid}:v2";
+ $images = implode("\n", [
+ "{$app->uuid}#v1#2026-01-01 00:00:00 +0000 UTC",
+ "{$app->uuid}#v2#2026-01-02 00:00:00 +0000 UTC",
+ "{$app->uuid}#v3#2026-01-03 00:00:00 +0000 UTC",
+ "{$app->uuid}#v4#2026-01-04 00:00:00 +0000 UTC",
+ "{$app->uuid}#pr-7#2026-01-05 00:00:00 +0000 UTC",
+ "{$app->uuid}#v1-build#2026-01-01 00:00:00 +0000 UTC",
+ "{$app->uuid}#v2-build#2026-01-02 00:00:00 +0000 UTC",
+ "{$app->uuid}#v4-build#2026-01-04 00:00:00 +0000 UTC",
+ "{$app->uuid}_web#v1#2026-01-01 00:00:00 +0000 UTC",
+ "{$app->uuid}/other#v1#2026-01-01 00:00:00 +0000 UTC",
+ ]);
+
+ $commands = collect();
+ Process::fake(function ($process) use ($commands, $containers, $images) {
+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+ $commands->push($command);
+
+ return match (true) {
+ str_contains($command, 'docker ps -a --format') => Process::result(output: $containers),
+ str_contains($command, "docker images --format '{{.Repository}}#") => Process::result(output: $images),
+ default => Process::result(output: ''),
+ };
+ });
+
+ CleanupDocker::run($server);
+
+ $applicationScanCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker ps -a --format')
+ || str_contains($command, "docker images --format '{{.Repository}}#")
+ || str_contains($command, 'docker inspect --format=')
+ || str_contains($command, "--filter reference='"));
+ $removals = $commands->filter(fn (string $command) => str_contains($command, 'docker rmi \''))->values();
+
+ expect($applicationScanCommands)->toHaveCount(2)
+ ->and($removals)->toHaveCount(1);
+
+ $removal = $removals->first();
+ foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build", "{$app->uuid}_web:v1"] as $deleted) {
+ expect($removal)->toContain("'{$deleted}'");
+ }
+ foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}/other"] as $kept) {
+ expect($removal)->not->toContain($kept);
+ }
+});