An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(deploy): start queued deployments atomically and report cancelled multi-server runs
Andras Bacsaif8f2945aec42
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.phpindex e9203cbab4..6a3101e1cc 100644--- a/app/Jobs/ApplicationDeploymentJob.php+++ b/app/Jobs/ApplicationDeploymentJob.php@@ -310,6 +310,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue$this->application_deployment_queue->refresh();if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {$this->application_deployment_queue->addLogEntry('Deployment was cancelled before starting.');+ $this->handleCancelledDeployment();return;}@@ -659,7 +660,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue/*** An inline Dockerfile image has no commit tag. The main server builds it and pushes it with the* `latest` tag just before it queues the additional servers, so an additional server pulls that- * image instead of building its own. A failed pull fails the deployment.+ * image instead of building its own. A missing image fails the deployment with a clear message.*/private function pull_image_for_additional_server(): bool{@@ -667,11 +668,28 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueuereturn false;}+ $image = escapeshellarg($this->production_image_name);$this->application_deployment_queue->addLogEntry("Pulling image ({$this->production_image_name}) that the main server pushed. Build step skipped.");- $this->execute_remote_command([- 'docker pull '.escapeshellarg($this->production_image_name),- 'hidden' => true,- ]);+ $this->execute_remote_command(+ [+ "docker pull {$image}",+ 'hidden' => true,+ 'ignore_errors' => true,+ ],+ [+ "docker images -q {$image} 2>/dev/null",+ 'hidden' => true,+ 'save' => 'local_image_found',+ ],+ );++ if (str($this->saved_outputs->get('local_image_found'))->isEmpty()) {+ $message = "Image ({$this->production_image_name}) not found in the registry; the main server must push it first.";+ if (! $this->server->canBuildApplications()) {+ $message .= " The server ({$this->server->name}) is set to deployments only and cannot build it.";+ }+ throw new DeploymentException($message);+ }return true;}@@ -5614,12 +5632,24 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {$this->application_deployment_queue->addLogEntry('Deployment cancelled by user, stopping execution.');+ $this->handleCancelledDeployment();throw new DeploymentException('Deployment cancelled by user', 69420);}return false;}+ /**+ * A cancelled deployment on an additional server can be the last one of a multi-server+ * deployment, so it must also trigger the combined notification.+ */+ private function handleCancelledDeployment(): void+ {+ if (filled($this->application_deployment_queue->parent_deployment_uuid)) {+ $this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);+ }+ }+/*** Update the deployment status in the database.*/diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.phpindex 8dc623af41..8fe4c0d393 100644--- a/bootstrap/helpers/applications.php+++ b/bootstrap/helpers/applications.php@@ -63,7 +63,7 @@ function queue_application_deployment(Application $application, string $deployme];}- return ApplicationDeploymentQueue::create([+ $deployment = ApplicationDeploymentQueue::create(['application_id' => $application_id,'application_name' => $application->name,'server_id' => $server_id,@@ -83,13 +83,24 @@ function queue_application_deployment(Application $application, string $deployme'only_this_server' => $only_this_server,'parent_deployment_uuid' => $parent_deployment_uuid,]);++ // Decide whether to start while the application and server rows are still locked,+ // so concurrent requests cannot both see an idle queue and start two deployments.+ $started = $no_questions_asked || next_queuable($server_id, $application_id, $commit, $pull_request_id);+ if ($started) {+ $deployment->update([+ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,+ ]);+ }++ return ['deployment' => $deployment, 'started' => $started];});- if (is_array($admission)) {+ if (! isset($admission['deployment'])) {return $admission;}- $deployment = $admission;+ $deployment = $admission['deployment'];if (auth()->check() && ! $is_webhook && ! $is_api && ! $rollback) {auditLog($restart_only ? 'ui.application.restarted' : 'ui.application.deployed', [@@ -100,17 +111,7 @@ function queue_application_deployment(Application $application, string $deployme]);}- if ($no_questions_asked) {- $deployment->update([- 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,- ]);- ApplicationDeploymentJob::dispatch(- application_deployment_queue_id: $deployment->id,- );- } elseif (next_queuable($server_id, $application_id, $commit, $pull_request_id)) {- $deployment->update([- 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,- ]);+ if ($admission['started']) {ApplicationDeploymentJob::dispatch(application_deployment_queue_id: $deployment->id,);@@ -141,17 +142,44 @@ function queue_next_deployment(Application $application)->sortBy('created_at');foreach ($queued_deployments as $next_deployment) {- // Check if this queued deployment can actually run- if (next_queuable($next_deployment->server_id, $next_deployment->application_id, $next_deployment->commit, $next_deployment->pull_request_id)) {- $next_deployment->update([- 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,- ]);+ start_queued_deployment($next_deployment);+ }+}- ApplicationDeploymentJob::dispatch(- application_deployment_queue_id: $next_deployment->id,- );+/**+ * Start a queued deployment if it can run now. The check and the status change run under+ * the same application and server row locks as queue admission, so two workers cannot both+ * start the same deployment or exceed the per-application and per-server limits.+ */+function start_queued_deployment(ApplicationDeploymentQueue $deployment): bool+{+ $started = DB::transaction(function () use ($deployment): bool {+ Application::query()->whereKey($deployment->application_id)->lockForUpdate()->first();+ Server::query()->whereKey($deployment->server_id)->lockForUpdate()->first();+ $current = ApplicationDeploymentQueue::query()->whereKey($deployment->id)->lockForUpdate()->first();++ if (! $current || $current->status !== ApplicationDeploymentStatus::QUEUED->value) {+ return false;+ }++ if (! next_queuable($current->server_id, $current->application_id, $current->commit, $current->pull_request_id)) {+ return false;}++ $current->update([+ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,+ ]);++ return true;+ });++ if ($started) {+ ApplicationDeploymentJob::dispatch(+ application_deployment_queue_id: $deployment->id,+ );}++ return $started;}function next_queuable(string $server_id, string $application_id, string $commit = 'HEAD', int $pull_request_id = 0): bool@@ -188,19 +216,8 @@ function next_after_cancel(?Server $server = null)->get()->sortBy('created_at');- if ($next_found->count() > 0) {- foreach ($next_found as $next) {- // Use next_queuable to properly check if this deployment can run- if (next_queuable($next->server_id, $next->application_id, $next->commit, $next->pull_request_id)) {- $next->update([- 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,- ]);-- ApplicationDeploymentJob::dispatch(- application_deployment_queue_id: $next->id,- );- }- }+ foreach ($next_found as $next) {+ start_queued_deployment($next);}}}diff --git a/tests/Feature/MultiServerApplicationTest.php b/tests/Feature/MultiServerApplicationTest.phpindex 13cac656f7..260c5747d7 100644--- a/tests/Feature/MultiServerApplicationTest.php+++ b/tests/Feature/MultiServerApplicationTest.php@@ -273,6 +273,46 @@ describe('combined deployment notification', function () {Notification::assertNotSentTo($this->team, DeploymentFailed::class);});+ test('sends the summary when the last additional server deployment is cancelled while running', function () {+ $parentUuid = new_public_id();+ Cache::put("multi-server-deployment-queued:{$parentUuid}", true);+ createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);+ $cancelled = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::CANCELLED_BY_USER->value, $parentUuid);+ $job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $cancelled]);++ expect(fn () => callMultiServerDeploymentJob($job, 'failDeployment'))+ ->toThrow(DeploymentException::class, 'Deployment cancelled by user');++ Notification::assertSentToTimes($this->team, DeploymentFailed::class, 1);+ Notification::assertSentTo($this->team, DeploymentFailed::class, fn (DeploymentFailed $notification) => $notification->deployment_uuid === $cancelled->deployment_uuid);+ Notification::assertNotSentTo($this->team, DeploymentSuccess::class);+ });++ test('sends the summary when the last additional server deployment is cancelled before it starts', function () {+ $parentUuid = new_public_id();+ Cache::put("multi-server-deployment-queued:{$parentUuid}", true);+ createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);+ $cancelled = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::CANCELLED_BY_USER->value, $parentUuid);+ [$job] = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $cancelled]);++ $job->handle();++ Notification::assertSentToTimes($this->team, DeploymentFailed::class, 1);+ Notification::assertSentTo($this->team, DeploymentFailed::class, fn (DeploymentFailed $notification) => $notification->deployment_uuid === $cancelled->deployment_uuid);+ });++ test('does not notify when a cancelled additional server deployment is not the last one', function () {+ $parentUuid = new_public_id();+ Cache::put("multi-server-deployment-queued:{$parentUuid}", true);+ createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::IN_PROGRESS->value, $parentUuid);+ $cancelled = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::CANCELLED_BY_USER->value, $parentUuid);+ [$job] = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $cancelled]);++ $job->handle();++ Notification::assertNothingSent();+ });+test('sends a failed notification that links to the failed server deployment', function () {$parentUuid = new_public_id();Cache::put("multi-server-deployment-queued:{$parentUuid}", true);@@ -297,15 +337,21 @@ class MultiServerRecordingDeploymentJob extends ApplicationDeploymentJobpublic function __construct() {}+ /** Output that `docker images -q` returns; empty means the image is missing. */+ public string $localImageOutput = 'sha256:0123456789ab';+public function execute_remote_command(...$commands): void{foreach ($commands as $command) {$this->recordedCommands[] = $command['command'] ?? $command[0];+ if (isset($command['save'])) {+ (new ReflectionProperty(ApplicationDeploymentJob::class, 'saved_outputs'))->getValue($this)->put($command['save'], $this->localImageOutput);+ }}}}-function makeInlineDockerfileJob(Application $application, bool $isAdditionalServer): array+function makeInlineDockerfileJob(Application $application, bool $isAdditionalServer, ?Server $server = null): array{$reflection = new ReflectionClass(ApplicationDeploymentJob::class);$job = new MultiServerRecordingDeploymentJob;@@ -316,6 +362,8 @@ function makeInlineDockerfileJob(Application $application, bool $isAdditionalSer'application_deployment_queue' => $queue,'is_this_additional_server' => $isAdditionalServer,'pull_request_id' => 0,+ 'saved_outputs' => collect(),+ 'server' => $server ?? $application->destination->server,] as $name => $value) {$reflection->getProperty($name)->setValue($job, $value);}@@ -336,7 +384,19 @@ describe('inline Dockerfile image on additional servers', function () {[$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), true);expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeTrue()- ->and($job->recordedCommands)->toBe(["docker pull 'registry.example.com/nginx-multi:latest'"]);+ ->and($job->recordedCommands)->toBe([+ "docker pull 'registry.example.com/nginx-multi:latest'",+ "docker images -q 'registry.example.com/nginx-multi:latest' 2>/dev/null",+ ]);+ });++ test('a deployments only server explains that the main server must push a missing image', function () {+ $this->extraServer->settings->update(['server_role' => 'deployment']);+ [$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), true, $this->extraServer->fresh());+ $job->localImageOutput = '';++ expect(fn () => $reflection->getMethod('pull_image_for_additional_server')->invoke($job))+ ->toThrow(DeploymentException::class, 'Image (registry.example.com/nginx-multi:latest) not found in the registry; the main server must push it first. The server (extra-server) is set to deployments only and cannot build it.');});test('the main server builds the image', function () {diff --git a/tests/Feature/QueueApplicationDeploymentCommitTest.php b/tests/Feature/QueueApplicationDeploymentCommitTest.phpindex 992856bae5..422047979b 100644--- a/tests/Feature/QueueApplicationDeploymentCommitTest.php+++ b/tests/Feature/QueueApplicationDeploymentCommitTest.php@@ -61,6 +61,62 @@ describe('queue_application_deployment commit resolution', function () {expect($creationLevel)->toBeGreaterThan($initialLevel);});+ test('starts a deployment only while the admission transaction is open', function () {+ $application = makeApplication($this->environment->id, $this->destination->id, 'HEAD');+ $initialLevel = DB::transactionLevel();+ $startLevel = null;++ ApplicationDeploymentQueue::updated(function (ApplicationDeploymentQueue $deployment) use (&$startLevel): void {+ if ($deployment->status === ApplicationDeploymentStatus::IN_PROGRESS->value) {+ $startLevel = DB::transactionLevel();+ }+ });++ $result = queue_application_deployment($application, 'atomic-start');++ expect($result['status'])->toBe('queued')+ ->and($startLevel)->toBeGreaterThan($initialLevel);+ Bus::assertDispatchedTimes(ApplicationDeploymentJob::class, 1);+ });++ test('starts the next queued deployment only while a transaction is open', function (string $trigger) {+ $application = makeApplication($this->environment->id, $this->destination->id, 'HEAD');+ $this->server->settings->update(['concurrent_builds' => 0]);+ queue_application_deployment($application, 'waiting-deployment');+ $this->server->settings->update(['concurrent_builds' => 1]);+ $initialLevel = DB::transactionLevel();+ $startLevel = null;++ ApplicationDeploymentQueue::updated(function (ApplicationDeploymentQueue $deployment) use (&$startLevel): void {+ if ($deployment->status === ApplicationDeploymentStatus::IN_PROGRESS->value) {+ $startLevel = DB::transactionLevel();+ }+ });++ $trigger === 'queue_next_deployment'+ ? queue_next_deployment($application)+ : next_after_cancel($this->server);++ expect($startLevel)->toBeGreaterThan($initialLevel)+ ->and(ApplicationDeploymentQueue::query()->sole()->status)->toBe(ApplicationDeploymentStatus::IN_PROGRESS->value);+ Bus::assertDispatchedTimes(ApplicationDeploymentJob::class, 1);+ })->with(['queue_next_deployment', 'next_after_cancel']);++ test('does not start a queued deployment that another worker already started', function () {+ $application = makeApplication($this->environment->id, $this->destination->id, 'HEAD');+ $this->server->settings->update(['concurrent_builds' => 0]);+ queue_application_deployment($application, 'stale-deployment');+ $this->server->settings->update(['concurrent_builds' => 5]);+ $stale = ApplicationDeploymentQueue::query()->sole();++ ApplicationDeploymentQueue::query()->whereKey($stale->id)->update([+ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,+ ]);++ expect(start_queued_deployment($stale))->toBeFalse();+ Bus::assertNotDispatched(ApplicationDeploymentJob::class);+ });+test('skips a matching commit without creating or dispatching a second deployment', function () {$application = makeApplication($this->environment->id, $this->destination->id, 'HEAD');