An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(deployments): advance the queue on all servers after a cancel
Andras Bacsai6cc6f1081df5
diff --git a/app/Actions/Application/CleanupPreviewDeployment.php b/app/Actions/Application/CleanupPreviewDeployment.phpindex 6c408bdf8b..661abc8d43 100644--- a/app/Actions/Application/CleanupPreviewDeployment.php+++ b/app/Actions/Application/CleanupPreviewDeployment.php@@ -56,7 +56,7 @@ class CleanupPreviewDeploymentif ($result['cancelled_deployments'] > 0) {try {- next_after_cancel($server);+ next_after_cancel($server, $application);} catch (\Throwable $e) {\Log::warning("Failed to advance deployment queue after cleaning up preview for application {$application->id}: {$e->getMessage()}");}diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.phpindex fc7a9d5460..5c405f3e17 100644--- a/app/Http/Controllers/Api/DeployController.php+++ b/app/Http/Controllers/Api/DeployController.php@@ -294,7 +294,7 @@ class DeployController extends Controller} finally {if ($cancelled) {try {- next_after_cancel($deploymentServer);+ next_after_cancel($deploymentServer, $deployment->application);} catch (\Throwable $e) {\Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}");}diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.phpindex a5df1ebfa4..52fa592766 100644--- a/app/Jobs/DeleteResourceJob.php+++ b/app/Jobs/DeleteResourceJob.php@@ -260,7 +260,7 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueueif ($cancelledDeployments > 0) {try {- next_after_cancel($server);+ next_after_cancel($server, $application);} catch (\Throwable $e) {\Log::warning("Failed to advance deployment queue after deleting preview {$this->resource->id}: {$e->getMessage()}");}diff --git a/app/Livewire/Project/Application/DeploymentNavbar.php b/app/Livewire/Project/Application/DeploymentNavbar.phpindex 22f2b8043d..654b571db3 100644--- a/app/Livewire/Project/Application/DeploymentNavbar.php+++ b/app/Livewire/Project/Application/DeploymentNavbar.php@@ -119,7 +119,7 @@ class DeploymentNavbar extends Component// Still mark as cancelled even if cleanup failsreturn handleError($e, $this);} finally {- next_after_cancel($server);+ next_after_cancel($server, $this->application);}}}diff --git a/app/Mcp/Tools/CancelDeployment.php b/app/Mcp/Tools/CancelDeployment.phpindex 5dcee3ed38..c9913c4e5f 100644--- a/app/Mcp/Tools/CancelDeployment.php+++ b/app/Mcp/Tools/CancelDeployment.php@@ -84,7 +84,7 @@ class CancelDeployment extends Tooltry {$deploymentServer = Server::whereTeamId($teamId)->find($deployment->server_id);- next_after_cancel($deploymentServer);+ next_after_cancel($deploymentServer, $application);} catch (\Throwable $e) {\Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}");}diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.phpindex 8b90de0f40..c2a012692f 100644--- a/bootstrap/helpers/applications.php+++ b/bootstrap/helpers/applications.php@@ -153,7 +153,7 @@ function queue_next_deployment(Application $application, ?int $finished_deployme->where('status', ApplicationDeploymentStatus::QUEUED)->distinct()->pluck('server_id');- $server_ids = collect([$application->destination->server_id, $finished_deployment_server_id])+ $server_ids = collect([$application->destination?->server_id, $finished_deployment_server_id])->merge($application_queued_server_ids)->filter(fn ($server_id) => $server_id !== null)->unique()@@ -164,7 +164,11 @@ function queue_next_deployment(Application $application, ?int $finished_deployme->sortBy('created_at');foreach ($queued_deployments as $next_deployment) {- start_queued_deployment($next_deployment);+ try {+ start_queued_deployment($next_deployment);+ } catch (Throwable $e) {+ Log::warning("Failed to start queued deployment {$next_deployment->deployment_uuid}: {$e->getMessage()}");+ }}}@@ -264,7 +268,11 @@ function next_queuable(string $server_id, string $application_id, string $commit}// Check server's concurrent build limit+ // A deleted server keeps its queued deployments; they cannot run anymore.$server = Server::find($server_id);+ if (! $server) {+ return false;+ }$concurrent_builds = $server->settings->concurrent_builds;$active_deployments = ApplicationDeploymentQueue::where('server_id', $server_id)->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)@@ -276,8 +284,17 @@ function next_queuable(string $server_id, string $application_id, string $commitreturn true;}-function next_after_cancel(?Server $server = null)+/**+ * Start the queued deployments that can run after a cancellation. With the application of the+ * cancelled deployment, its queued deployments on all servers start too, as when a deployment ends.+ */+function next_after_cancel(?Server $server = null, ?Application $application = null){+ if ($application) {+ queue_next_deployment($application, $server?->id);++ return;+ }if ($server) {$next_found = ApplicationDeploymentQueue::where('server_id', data_get($server, 'id'))->where('status', ApplicationDeploymentStatus::QUEUED)diff --git a/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php b/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.phpnew file mode 100644index 0000000000..aac9f6c8d9--- /dev/null+++ b/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php@@ -0,0 +1,136 @@+<?php++use App\Enums\ApplicationDeploymentStatus;+use App\Jobs\ApplicationDeploymentJob;+use App\Livewire\Project\Application\DeploymentNavbar;+use App\Models\Application;+use App\Models\ApplicationDeploymentQueue;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Log;+use Illuminate\Support\Facades\Notification;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Queue;+use Illuminate\Support\Facades\Storage;+use Livewire\Livewire;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Queue::fake();+ Notification::fake();+ Storage::fake('ssh-keys');+ Storage::fake('ssh-mux');+ Process::fake(['*' => Process::result(output: '')]);+ config([+ 'constants.ssh.mux_enabled' => false,+ 'cache.default' => 'array',+ 'app.maintenance.store' => 'array',+ ]);++ $settings = new InstanceSettings(['is_api_enabled' => true, 'is_mcp_server_enabled' => true]);+ $settings->id = 0;+ $settings->save();++ $this->team = Team::factory()->create();+ $this->user = User::factory()->create();+ $this->team->members()->attach($this->user->id, ['role' => 'owner']);+ session(['currentTeam' => $this->team]);++ $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);+ $makeServer = function (string $ip) use ($privateKey): Server {+ $server = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $privateKey->id, 'ip' => $ip]);+ $server->settings->update(['concurrent_builds' => 1]);++ return $server;+ };+ $this->primaryServer = $makeServer('203.0.113.31');+ $this->additionalServer = $makeServer('203.0.113.32');+ $this->deletedServer = $makeServer('203.0.113.33');++ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $primaryDestination = StandaloneDocker::where('server_id', $this->primaryServer->id)->firstOrFail();+ $this->application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $primaryDestination->id,+ 'destination_type' => $primaryDestination->getMorphClass(),+ ]);+ foreach ([$this->additionalServer, $this->deletedServer] as $server) {+ $this->application->additional_servers()->attach($server->id, [+ 'standalone_docker_id' => StandaloneDocker::where('server_id', $server->id)->firstOrFail()->id,+ ]);+ }++ $this->makeQueueDeployment = fn (Server $server, string $status, string $uuid): ApplicationDeploymentQueue => ApplicationDeploymentQueue::create([+ 'application_id' => $this->application->id,+ 'deployment_uuid' => $uuid,+ 'status' => $status,+ 'server_id' => $server->id,+ 'destination_id' => StandaloneDocker::where('server_id', $server->id)->firstOrFail()->id,+ 'commit' => 'HEAD',+ 'pull_request_id' => 0,+ 'only_this_server' => true,+ ]);+});++afterEach(function () {+ Server::flushIdentityMap();+});++test('a finished deployment starts the queued deployment on another server when a deleted server also has one queued', function () {+ $running = ($this->makeQueueDeployment)($this->primaryServer, 'in_progress', 'queue-across-running');+ $orphaned = ($this->makeQueueDeployment)($this->deletedServer, 'queued', 'queue-across-orphaned');+ $this->travel(1)->seconds();+ $waiting = ($this->makeQueueDeployment)($this->additionalServer, 'queued', 'queue-across-waiting');+ // A server deleted before deletion failed its queued deployments left them queued.+ Server::withoutEvents(fn () => $this->deletedServer->delete());+ Server::flushIdentityMap();+ Log::spy();++ (new ApplicationDeploymentJob($running->id))->failed(new RuntimeException('build failed'));++ Log::shouldNotHaveReceived('warning', [Mockery::pattern('/Starting the next queued deployment failed/')]);+ expect($waiting->fresh()->status)->toBe('in_progress')+ ->and($orphaned->fresh()->status)->toBe('queued');+ Queue::assertPushed(ApplicationDeploymentJob::class, 1);+ Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waiting->id);+});++test('cancelling a deployment starts the queued deployment of the same application on another server', function (string $path) {+ $running = ($this->makeQueueDeployment)($this->primaryServer, 'in_progress', 'queue-across-running');+ $waiting = ($this->makeQueueDeployment)($this->additionalServer, 'queued', 'queue-across-waiting');++ if ($path === 'ui') {+ $this->actingAs($this->user);+ Livewire::test(DeploymentNavbar::class, ['application_deployment_queue' => $running])->call('cancel');+ } elseif ($path === 'api') {+ $token = $this->user->createToken('queue-across', ['*'])->plainTextToken;+ $this->withHeaders(['Authorization' => 'Bearer '.$token])+ ->postJson("/api/v1/deployments/{$running->deployment_uuid}/cancel")+ ->assertSuccessful();+ } else {+ $token = $this->user->createToken('queue-across', ['read', 'deploy'])->plainTextToken;+ $this->withHeaders([+ 'Accept' => 'application/json, text/event-stream',+ 'Authorization' => 'Bearer '.$token,+ ])->postJson('/mcp', [+ 'jsonrpc' => '2.0',+ 'id' => 1,+ 'method' => 'tools/call',+ 'params' => ['name' => 'cancel_deployment', 'arguments' => ['uuid' => $running->deployment_uuid]],+ ])->assertOk();+ }++ expect($running->fresh()->status)->toBe(ApplicationDeploymentStatus::CANCELLED_BY_USER->value)+ ->and($waiting->fresh()->status)->toBe(ApplicationDeploymentStatus::IN_PROGRESS->value);+ Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waiting->id);+})->with(['ui', 'api', 'mcp']);