Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(deployments): advance the queue on all servers after a cancel

Andras Bacsai6cc6f1081df5
diff --git a/app/Actions/Application/CleanupPreviewDeployment.php b/app/Actions/Application/CleanupPreviewDeployment.php
index 6c408bdf8b..661abc8d43 100644
--- a/app/Actions/Application/CleanupPreviewDeployment.php
+++ b/app/Actions/Application/CleanupPreviewDeployment.php
@@ -56,7 +56,7 @@ class CleanupPreviewDeployment
if ($result['cancelled_deployments'] > 0) {
try {
- next_after_cancel($server);
+ next_after_cancel($server, $application);
} catch (\Throwable $e) {
\Log::warning("Failed to advance deployment queue after cleaning up preview for application {$application->id}: {$e->getMessage()}");
}
diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php
index fc7a9d5460..5c405f3e17 100644
--- a/app/Http/Controllers/Api/DeployController.php
+++ b/app/Http/Controllers/Api/DeployController.php
@@ -294,7 +294,7 @@ class DeployController extends Controller
} finally {
if ($cancelled) {
try {
- next_after_cancel($deploymentServer);
+ next_after_cancel($deploymentServer, $deployment->application);
} catch (\Throwable $e) {
\Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}");
}
diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php
index a5df1ebfa4..52fa592766 100644
--- a/app/Jobs/DeleteResourceJob.php
+++ b/app/Jobs/DeleteResourceJob.php
@@ -260,7 +260,7 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
if ($cancelledDeployments > 0) {
try {
- next_after_cancel($server);
+ next_after_cancel($server, $application);
} catch (\Throwable $e) {
\Log::warning("Failed to advance deployment queue after deleting preview {$this->resource->id}: {$e->getMessage()}");
}
diff --git a/app/Livewire/Project/Application/DeploymentNavbar.php b/app/Livewire/Project/Application/DeploymentNavbar.php
index 22f2b8043d..654b571db3 100644
--- a/app/Livewire/Project/Application/DeploymentNavbar.php
+++ b/app/Livewire/Project/Application/DeploymentNavbar.php
@@ -119,7 +119,7 @@ class DeploymentNavbar extends Component
// Still mark as cancelled even if cleanup fails
return handleError($e, $this);
} finally {
- next_after_cancel($server);
+ next_after_cancel($server, $this->application);
}
}
}
diff --git a/app/Mcp/Tools/CancelDeployment.php b/app/Mcp/Tools/CancelDeployment.php
index 5dcee3ed38..c9913c4e5f 100644
--- a/app/Mcp/Tools/CancelDeployment.php
+++ b/app/Mcp/Tools/CancelDeployment.php
@@ -84,7 +84,7 @@ class CancelDeployment extends Tool
try {
$deploymentServer = Server::whereTeamId($teamId)->find($deployment->server_id);
- next_after_cancel($deploymentServer);
+ next_after_cancel($deploymentServer, $application);
} catch (\Throwable $e) {
\Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}");
}
diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php
index 8b90de0f40..c2a012692f 100644
--- a/bootstrap/helpers/applications.php
+++ b/bootstrap/helpers/applications.php
@@ -153,7 +153,7 @@ function queue_next_deployment(Application $application, ?int $finished_deployme
->where('status', ApplicationDeploymentStatus::QUEUED)
->distinct()
->pluck('server_id');
- $server_ids = collect([$application->destination->server_id, $finished_deployment_server_id])
+ $server_ids = collect([$application->destination?->server_id, $finished_deployment_server_id])
->merge($application_queued_server_ids)
->filter(fn ($server_id) => $server_id !== null)
->unique()
@@ -164,7 +164,11 @@ function queue_next_deployment(Application $application, ?int $finished_deployme
->sortBy('created_at');
foreach ($queued_deployments as $next_deployment) {
- start_queued_deployment($next_deployment);
+ try {
+ start_queued_deployment($next_deployment);
+ } catch (Throwable $e) {
+ Log::warning("Failed to start queued deployment {$next_deployment->deployment_uuid}: {$e->getMessage()}");
+ }
}
}
@@ -264,7 +268,11 @@ function next_queuable(string $server_id, string $application_id, string $commit
}
// Check server's concurrent build limit
+ // A deleted server keeps its queued deployments; they cannot run anymore.
$server = Server::find($server_id);
+ if (! $server) {
+ return false;
+ }
$concurrent_builds = $server->settings->concurrent_builds;
$active_deployments = ApplicationDeploymentQueue::where('server_id', $server_id)
->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)
@@ -276,8 +284,17 @@ function next_queuable(string $server_id, string $application_id, string $commit
return true;
}
-function next_after_cancel(?Server $server = null)
+/**
+ * Start the queued deployments that can run after a cancellation. With the application of the
+ * cancelled deployment, its queued deployments on all servers start too, as when a deployment ends.
+ */
+function next_after_cancel(?Server $server = null, ?Application $application = null)
{
+ if ($application) {
+ queue_next_deployment($application, $server?->id);
+
+ return;
+ }
if ($server) {
$next_found = ApplicationDeploymentQueue::where('server_id', data_get($server, 'id'))
->where('status', ApplicationDeploymentStatus::QUEUED)
diff --git a/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php b/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php
new file mode 100644
index 0000000000..aac9f6c8d9
--- /dev/null
+++ b/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php
@@ -0,0 +1,136 @@
+<?php
+
+use App\Enums\ApplicationDeploymentStatus;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Livewire\Project\Application\DeploymentNavbar;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Queue::fake();
+ Notification::fake();
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ Process::fake(['*' => Process::result(output: '')]);
+ config([
+ 'constants.ssh.mux_enabled' => false,
+ 'cache.default' => 'array',
+ 'app.maintenance.store' => 'array',
+ ]);
+
+ $settings = new InstanceSettings(['is_api_enabled' => true, 'is_mcp_server_enabled' => true]);
+ $settings->id = 0;
+ $settings->save();
+
+ $this->team = Team::factory()->create();
+ $this->user = User::factory()->create();
+ $this->team->members()->attach($this->user->id, ['role' => 'owner']);
+ session(['currentTeam' => $this->team]);
+
+ $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
+ $makeServer = function (string $ip) use ($privateKey): Server {
+ $server = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $privateKey->id, 'ip' => $ip]);
+ $server->settings->update(['concurrent_builds' => 1]);
+
+ return $server;
+ };
+ $this->primaryServer = $makeServer('203.0.113.31');
+ $this->additionalServer = $makeServer('203.0.113.32');
+ $this->deletedServer = $makeServer('203.0.113.33');
+
+ $project = Project::factory()->create(['team_id' => $this->team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $primaryDestination = StandaloneDocker::where('server_id', $this->primaryServer->id)->firstOrFail();
+ $this->application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $primaryDestination->id,
+ 'destination_type' => $primaryDestination->getMorphClass(),
+ ]);
+ foreach ([$this->additionalServer, $this->deletedServer] as $server) {
+ $this->application->additional_servers()->attach($server->id, [
+ 'standalone_docker_id' => StandaloneDocker::where('server_id', $server->id)->firstOrFail()->id,
+ ]);
+ }
+
+ $this->makeQueueDeployment = fn (Server $server, string $status, string $uuid): ApplicationDeploymentQueue => ApplicationDeploymentQueue::create([
+ 'application_id' => $this->application->id,
+ 'deployment_uuid' => $uuid,
+ 'status' => $status,
+ 'server_id' => $server->id,
+ 'destination_id' => StandaloneDocker::where('server_id', $server->id)->firstOrFail()->id,
+ 'commit' => 'HEAD',
+ 'pull_request_id' => 0,
+ 'only_this_server' => true,
+ ]);
+});
+
+afterEach(function () {
+ Server::flushIdentityMap();
+});
+
+test('a finished deployment starts the queued deployment on another server when a deleted server also has one queued', function () {
+ $running = ($this->makeQueueDeployment)($this->primaryServer, 'in_progress', 'queue-across-running');
+ $orphaned = ($this->makeQueueDeployment)($this->deletedServer, 'queued', 'queue-across-orphaned');
+ $this->travel(1)->seconds();
+ $waiting = ($this->makeQueueDeployment)($this->additionalServer, 'queued', 'queue-across-waiting');
+ // A server deleted before deletion failed its queued deployments left them queued.
+ Server::withoutEvents(fn () => $this->deletedServer->delete());
+ Server::flushIdentityMap();
+ Log::spy();
+
+ (new ApplicationDeploymentJob($running->id))->failed(new RuntimeException('build failed'));
+
+ Log::shouldNotHaveReceived('warning', [Mockery::pattern('/Starting the next queued deployment failed/')]);
+ expect($waiting->fresh()->status)->toBe('in_progress')
+ ->and($orphaned->fresh()->status)->toBe('queued');
+ Queue::assertPushed(ApplicationDeploymentJob::class, 1);
+ Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waiting->id);
+});
+
+test('cancelling a deployment starts the queued deployment of the same application on another server', function (string $path) {
+ $running = ($this->makeQueueDeployment)($this->primaryServer, 'in_progress', 'queue-across-running');
+ $waiting = ($this->makeQueueDeployment)($this->additionalServer, 'queued', 'queue-across-waiting');
+
+ if ($path === 'ui') {
+ $this->actingAs($this->user);
+ Livewire::test(DeploymentNavbar::class, ['application_deployment_queue' => $running])->call('cancel');
+ } elseif ($path === 'api') {
+ $token = $this->user->createToken('queue-across', ['*'])->plainTextToken;
+ $this->withHeaders(['Authorization' => 'Bearer '.$token])
+ ->postJson("/api/v1/deployments/{$running->deployment_uuid}/cancel")
+ ->assertSuccessful();
+ } else {
+ $token = $this->user->createToken('queue-across', ['read', 'deploy'])->plainTextToken;
+ $this->withHeaders([
+ 'Accept' => 'application/json, text/event-stream',
+ 'Authorization' => 'Bearer '.$token,
+ ])->postJson('/mcp', [
+ 'jsonrpc' => '2.0',
+ 'id' => 1,
+ 'method' => 'tools/call',
+ 'params' => ['name' => 'cancel_deployment', 'arguments' => ['uuid' => $running->deployment_uuid]],
+ ])->assertOk();
+ }
+
+ expect($running->fresh()->status)->toBe(ApplicationDeploymentStatus::CANCELLED_BY_USER->value)
+ ->and($waiting->fresh()->status)->toBe(ApplicationDeploymentStatus::IN_PROGRESS->value);
+ Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waiting->id);
+})->with(['ui', 'api', 'mcp']);