Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(multi-server): make applications on several servers consistent

Andras Bacsai7bd8baa7f056
diff --git a/.ai/lessons.md b/.ai/lessons.md
index f47d1c0f63..79bac6eff3 100644
--- a/.ai/lessons.md
+++ b/.ai/lessons.md
@@ -24,6 +24,7 @@
- A source-text check is acceptable only for a correctness or security rule that rendering cannot prove easily, such as stable `wire:key` values or no raw user output.
- Call the app code under test. Do not copy app logic into a test, assert only literals, constants, `class_exists`/`method_exists`, or PHP built-ins such as `escapeshellarg`.
- Pest test files share one global function scope. Give file-level helper functions a unique name.
+- Call `Server::flushIdentityMap()` in `beforeEach` of tests that use `Server` models; the identity map leaks servers and settings between test files in one run.
## Keep dynamic Livewire identities stable
- In dynamic lists, key components and actions with immutable record identities, not counts, indexes, or array positions.
diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php
index 14291d5f91..a0e50ede79 100644
--- a/app/Actions/Application/StopApplication.php
+++ b/app/Actions/Application/StopApplication.php
@@ -20,10 +20,13 @@ class StopApplication
if ($application?->additional_servers?->count() > 0) {
$servers = $servers->merge($application->additional_servers);
}
+ $errors = [];
foreach ($servers as $server) {
try {
if (! $server->isFunctional()) {
- return 'Server is not functional';
+ $errors[] = "Server {$server->name} is not functional.";
+
+ continue;
}
if ($server->isSwarm()) {
@@ -58,9 +61,12 @@ class StopApplication
CleanupDocker::dispatch($server, false, false);
}
} catch (\Exception $e) {
- return $e->getMessage();
+ $errors[] = $e->getMessage();
}
}
+ if ($errors !== []) {
+ return implode(' ', $errors);
+ }
$status = [
'status' => 'exited',
diff --git a/app/Actions/Shared/EnsureContentFilesOnServer.php b/app/Actions/Shared/EnsureContentFilesOnServer.php
index 847665f1df..a23b5fb9d3 100644
--- a/app/Actions/Shared/EnsureContentFilesOnServer.php
+++ b/app/Actions/Shared/EnsureContentFilesOnServer.php
@@ -66,7 +66,7 @@ class EnsureContentFilesOnServer
$written = 0;
foreach ($toWrite as [$fileStorage, $path]) {
try {
- $fileStorage->saveStorageOnServer();
+ $fileStorage->saveStorageOnServer($server);
$written++;
} catch (\Throwable $e) {
$log("Warning: Coolify cannot write the configuration file {$path}: ".$this->plainText($e->getMessage()), 'stderr');
diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php
index 04b574abe9..128744a549 100644
--- a/app/Http/Controllers/Api/ApplicationsController.php
+++ b/app/Http/Controllers/Api/ApplicationsController.php
@@ -5287,6 +5287,10 @@ class ApplicationsController extends Controller
], 422);
}
+ if ($reason = $application->persistentStorageUnavailableReason()) {
+ return response()->json(['message' => $reason], 422);
+ }
+
$storage = LocalPersistentVolume::create([
'name' => $application->uuid.'-'.$request->name,
'mount_path' => $request->mount_path,
@@ -6178,6 +6182,10 @@ class ApplicationsController extends Controller
return response()->json(['message' => 'A destination on this server is already attached.'], 422);
}
+ if ($reason = $application->additionalServersUnavailableReason()) {
+ return response()->json(['message' => $reason], 422);
+ }
+
$application->additional_networks()->attach($destination->id, ['server_id' => $destination->server_id]);
auditLog('api.application.destination_added', [
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php
index 0b69aff669..6ff61ad9c1 100644
--- a/app/Jobs/ApplicationDeploymentJob.php
+++ b/app/Jobs/ApplicationDeploymentJob.php
@@ -34,6 +34,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Sleep;
use Illuminate\Support\Str;
use JsonException;
@@ -315,6 +316,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
try {
$this->validateDeploymentEnvironmentVariableKeys();
+ $this->ensureRegistryImageForMultipleServers();
+ $this->warnAboutVolumesOnMultipleServers();
} catch (Exception $e) {
$this->fail($e);
throw $e;
@@ -619,6 +622,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
);
$this->generate_image_names();
$this->generate_compose_file();
+ if ($this->pull_image_for_additional_server()) {
+ $this->save_runtime_environment_variables();
+ $this->rolling_update();
+
+ return;
+ }
// Save build-time .env file BEFORE the build
$this->save_buildtime_environment_variables();
@@ -635,6 +644,26 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->rolling_update();
}
+ /**
+ * An inline Dockerfile image has no commit tag. The main server builds it and pushes it with the
+ * `latest` tag just before it queues the additional servers, so an additional server pulls that
+ * image instead of building its own. A failed pull fails the deployment.
+ */
+ private function pull_image_for_additional_server(): bool
+ {
+ if (! $this->is_this_additional_server || str($this->application->docker_registry_image_name)->isEmpty()) {
+ return false;
+ }
+
+ $this->application_deployment_queue->addLogEntry("Pulling image ({$this->production_image_name}) that the main server pushed. Build step skipped.");
+ $this->execute_remote_command([
+ 'docker pull '.escapeshellarg($this->production_image_name),
+ 'hidden' => true,
+ ]);
+
+ return true;
+ }
+
private function deploy_dockerimage_buildpack()
{
$this->dockerImage = $this->application->docker_registry_image_name;
@@ -715,14 +744,14 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$fileStorage->is_directory = true;
$fileStorage->content = null;
$fileStorage->save();
- $fileStorage->deleteStorageOnServer();
- $fileStorage->saveStorageOnServer();
+ $fileStorage->deleteStorageOnServer($this->mainServer);
+ $fileStorage->saveStorageOnServer($this->mainServer);
} elseif ($fileStat->value() === 'regular file' && $fileStorage->is_directory) {
$fileStorage->is_directory = false;
$fileStorage->is_based_on_git = true;
$fileStorage->save();
- $fileStorage->deleteStorageOnServer();
- $fileStorage->saveStorageOnServer();
+ $fileStorage->deleteStorageOnServer($this->mainServer);
+ $fileStorage->saveStorageOnServer($this->mainServer);
}
}
}
@@ -1002,16 +1031,13 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
/**
* The queued ServerStorageSaveJob writes new content files, but it can run after the containers
* start. Preserve-repository deployments write all file storages in write_deployment_configurations().
- * Coolify writes file storages only on the main server of the application.
+ * Each deployment writes the files on its own server, so additional servers also get them.
*/
private function write_missing_content_files(): void
{
if ($this->preserveRepository) {
return;
}
- if ((int) data_get($this->application, 'destination.server_id') !== (int) $this->mainServer->id) {
- return;
- }
EnsureContentFilesOnServer::run(
$this->application->fileStorages()->get(),
@@ -1173,7 +1199,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
foreach ($this->application->fileStorages as $fileStorage) {
if (! $fileStorage->is_host_file && ! $fileStorage->is_based_on_git && ! $fileStorage->is_directory) {
- $fileStorage->saveStorageOnServer();
+ $fileStorage->saveStorageOnServer($this->mainServer);
}
}
if ($this->use_build_server) {
@@ -1216,7 +1242,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function push_to_docker_registry()
{
- $forceFail = true;
if (str($this->application->docker_registry_image_name)->isEmpty()) {
return;
}
@@ -1226,15 +1251,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
if ($this->application->build_pack === 'dockerimage') {
return;
}
- if ($this->use_build_server) {
- $forceFail = true;
- }
- if ($this->server->isSwarm() && $this->build_pack !== 'dockerimage') {
- $forceFail = true;
- }
- if ($this->application->additional_servers->count() > 0) {
- $forceFail = true;
- }
if ($this->is_this_additional_server) {
return;
}
@@ -1266,9 +1282,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
} catch (Exception $e) {
$this->application_deployment_queue->addLogEntry('Failed to push image to docker registry. Please check debug logs for more information.');
- if ($forceFail) {
- throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
- }
+ throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
}
}
@@ -2139,6 +2153,42 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ /**
+ * Coolify blocks new volumes on multi-server applications, but older applications can have both.
+ * Some applications accept separate data on each server (for example a cache), so the
+ * deployment continues and only shows a warning.
+ */
+ private function warnAboutVolumesOnMultipleServers(): void
+ {
+ if ($this->pull_request_id !== 0) {
+ return;
+ }
+ if ($this->application->additional_servers()->doesntExist() || $this->application->persistentStorages()->doesntExist()) {
+ return;
+ }
+
+ $this->application_deployment_queue->addLogEntry('Warning: This application uses multiple servers and has persistent storage. Volumes are not shared between servers, so each server has its own data.', 'stderr');
+ }
+
+ /**
+ * Additional servers pull the image that the main server pushes. Without a registry image, each
+ * server builds its own image, so the servers can run different code.
+ */
+ private function ensureRegistryImageForMultipleServers(): void
+ {
+ if ($this->pull_request_id !== 0) {
+ return;
+ }
+ if ($this->application->additional_servers()->doesntExist()) {
+ return;
+ }
+ if (str($this->application->docker_registry_image_name)->isNotEmpty()) {
+ return;
+ }
+
+ throw new DeploymentException('Before deploying to multiple servers, you must set a Docker image in the General tab. More information: https://coolify.io/docs/knowledge-base/server/multiple-servers');
+ }
+
/**
* Build-time names go into shell and Docker build commands, so they must be valid when the
* deployment builds an image. Runtime-only variables only go into the .env file: existing ones
@@ -2640,23 +2690,27 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->prepare_builder_image(firstTry: false);
}
- private function deploy_to_additional_destinations()
+ /**
+ * @return int The number of deployments that Coolify queued on additional servers
+ */
+ private function deploy_to_additional_destinations(): int
{
if ($this->application->additional_networks->count() === 0) {
- return;
+ return 0;
}
if ($this->pull_request_id !== 0) {
- return;
+ return 0;
}
$destination_ids = $this->application->additional_networks->pluck('id');
if ($this->server->isSwarm()) {
$this->application_deployment_queue->addLogEntry('Additional destinations are not supported in swarm mode.');
- return;
+ return 0;
}
if ($destination_ids->contains($this->destination->id)) {
- return;
+ return 0;
}
+ $queued = 0;
foreach ($destination_ids as $destination_id) {
$destination = StandaloneDocker::find($destination_id);
if (! $destination) {
@@ -2669,16 +2723,29 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
continue;
}
$deployment_uuid = new_public_id();
- queue_application_deployment(
+ // Deploy the exact commit and image tag of the main server, also for a rollback.
+ $result = queue_application_deployment(
deployment_uuid: $deployment_uuid,
application: $this->application,
+ commit: $this->commit,
+ no_questions_asked: true,
server: $server,
destination: $destination,
- no_questions_asked: true,
+ rollback: $this->rollback,
+ docker_registry_image_tag: $this->application_deployment_queue->docker_registry_image_tag,
+ parent_deployment_uuid: $this->deployment_uuid,
);
+ if ($result['status'] !== 'queued') {
+ $this->application_deployment_queue->addLogEntry("Deployment to {$server->name} was not queued: {$result['message']}", 'stderr');
+
+ continue;
+ }
+ $queued++;
$deployment_url = base_url().'/project/'.data_get($this->application, 'environment.project.uuid').'/environment/'.data_get($this->application, 'environment.uuid').'/application/'.data_get($this->application, 'uuid')."/deployment/{$deployment_uuid}";
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: {$deployment_url}");
}
+
+ return $queued;
}
private function set_coolify_variables()
@@ -5547,11 +5614,20 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
event(new ApplicationConfigurationChanged($this->application->team()->id));
- if (! $this->only_this_server) {
- $this->deploy_to_additional_destinations();
+ if (filled($this->application_deployment_queue->parent_deployment_uuid)) {
+ $this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);
+
+ return;
}
- $this->sendDeploymentNotification(DeploymentSuccess::class);
+ $queuedAdditionalDeployments = $this->only_this_server ? 0 : $this->deploy_to_additional_destinations();
+ if ($queuedAdditionalDeployments === 0) {
+ $this->sendDeploymentNotification(DeploymentSuccess::class);
+
+ return;
+ }
+ Cache::put("multi-server-deployment-queued:{$this->deployment_uuid}", true, now()->addDay());
+ $this->sendMultiServerDeploymentNotification($this->deployment_uuid);
}
/**
@@ -5559,9 +5635,47 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
*/
private function handleFailedDeployment(): void
{
+ if (filled($this->application_deployment_queue->parent_deployment_uuid)) {
+ $this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);
+
+ return;
+ }
+
$this->sendDeploymentNotification(DeploymentFailed::class);
}
+ /**
+ * The main server queues one deployment for each additional server. When all are queued and the
+ * last one ends, Coolify sends one notification: success if all servers succeeded, else failed
+ * with a link to the first deployment that did not succeed.
+ */
+ private function sendMultiServerDeploymentNotification(string $parentDeploymentUuid): void
+ {
+ if (! Cache::has("multi-server-deployment-queued:{$parentDeploymentUuid}")) {
+ return;
+ }
+ $deployments = ApplicationDeploymentQueue::query()
+ ->where('parent_deployment_uuid', $parentDeploymentUuid)
+ ->get(['deployment_uuid', 'status']);
+
+ $endStatuses = [
+ ApplicationDeploymentStatus::FINISHED->value,
+ ApplicationDeploymentStatus::FAILED->value,
+ ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
+ ];
+ if ($deployments->contains(fn (ApplicationDeploymentQueue $deployment) => ! in_array($deployment->status, $endStatuses, true))) {
+ return;
+ }
+ if (! Cache::add("multi-server-deployment-notified:{$parentDeploymentUuid}", true, now()->addDay())) {
+ return;
+ }
+
+ $unsuccessful = $deployments->first(fn (ApplicationDeploymentQueue $deployment) => $deployment->status !== ApplicationDeploymentStatus::FINISHED->value);
+ $this->application->environment->project->team?->notify($unsuccessful
+ ? new DeploymentFailed($this->application, $unsuccessful->deployment_uuid, $this->preview)
+ : new DeploymentSuccess($this->application, $parentDeploymentUuid, $this->preview));
+ }
+
/**
* Send deployment status notification to the team.
*/
diff --git a/app/Livewire/Project/Database/Sqlite/ConnectApplication.php b/app/Livewire/Project/Database/Sqlite/ConnectApplication.php
index 0c5620189b..d336c162ce 100644
--- a/app/Livewire/Project/Database/Sqlite/ConnectApplication.php
+++ b/app/Livewire/Project/Database/Sqlite/ConnectApplication.php
@@ -97,6 +97,10 @@ class ConnectApplication extends Component
return null;
}
+ if ($reason = $application->persistentStorageUnavailableReason()) {
+ throw new \Exception($reason);
+ }
+
if ($application->persistentStorages()->where('standalone_sqlite_id', $this->database->id)->exists()) {
throw new \Exception("{$application->name} already mounts this database volume.");
}
diff --git a/app/Livewire/Project/Service/Storage.php b/app/Livewire/Project/Service/Storage.php
index 785c5531df..6bb1d3a588 100644
--- a/app/Livewire/Project/Service/Storage.php
+++ b/app/Livewire/Project/Service/Storage.php
@@ -240,6 +240,14 @@ class Storage extends Component
return collect($this->fileStorage)->where('is_directory', true);
}
+ /**
+ * The reason why the resource cannot get a volume mount, or null if it can.
+ */
+ public function getVolumeUnavailableReasonProperty(): ?string
+ {
+ return $this->resource instanceof Application ? $this->resource->persistentStorageUnavailableReason() : null;
+ }
+
public function getVolumeCountProperty()
{
return $this->cachedVolumeCount;
@@ -265,6 +273,12 @@ class Storage extends Component
'mount_path' => 'required|string',
], ValidationPatterns::volumeNameMessages());
+ if ($reason = $this->volumeUnavailableReason) {
+ $this->dispatch('error', 'Failed to add volume.', $reason);
+
+ return;
+ }
+
$name = $this->resource->uuid.'-'.$this->name;
LocalPersistentVolume::create([
diff --git a/app/Livewire/Project/Shared/Destination.php b/app/Livewire/Project/Shared/Destination.php
index 4118c0c7ab..65af4b5ab7 100644
--- a/app/Livewire/Project/Shared/Destination.php
+++ b/app/Livewire/Project/Shared/Destination.php
@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Shared;
use App\Actions\Application\StopApplicationOneServer;
use App\Actions\Docker\GetContainersStatus;
use App\Events\ApplicationStatusChanged;
+use App\Models\Application;
use App\Models\Server;
use App\Models\StandaloneDocker;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
@@ -156,6 +157,14 @@ class Destination extends Component
$server = Server::ownedByCurrentTeam()->findOrFail($server_id);
$network = StandaloneDocker::ownedByCurrentTeam()->where('server_id', $server->id)->findOrFail($network_id);
$this->authorize('update', $this->resource);
+ $reason = $this->resource instanceof Application
+ ? $this->resource->additionalServersUnavailableReason()
+ : 'Only applications can use multiple servers.';
+ if ($reason) {
+ $this->dispatch('error', 'Failed to add server.', $reason);
+
+ return;
+ }
$this->resource->additional_networks()->syncWithoutDetaching([
$network->id => ['server_id' => $server->id],
diff --git a/app/Models/Application.php b/app/Models/Application.php
index 5d3dd8bb2c..987a752605 100644
--- a/app/Models/Application.php
+++ b/app/Models/Application.php
@@ -585,6 +585,33 @@ class Application extends BaseModel
->withPivot('server_id', 'status');
}
+ /**
+ * The reason why this application cannot get a persistent volume, or null if it can.
+ */
+ public function persistentStorageUnavailableReason(): ?string
+ {
+ if ($this->additional_servers()->exists()) {
+ return 'Applications that use multiple servers cannot have persistent storage because volumes are not shared between servers.';
+ }
+
+ return null;
+ }
+
+ /**
+ * The reason why this application cannot deploy to additional servers, or null if it can.
+ */
+ public function additionalServersUnavailableReason(): ?string
+ {
+ if ($this->build_pack === 'dockercompose') {
+ return 'Docker Compose applications cannot use multiple servers.';
+ }
+ if ($this->persistentStorages()->exists()) {
+ return 'Applications with persistent storage cannot use multiple servers because volumes are not shared between servers.';
+ }
+
+ return null;
+ }
+
public function is_public_repository(): bool
{
if (data_get($this, 'source.is_public')) {
diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php
index 451136b5f4..bc844d826d 100644
--- a/app/Models/ApplicationDeploymentQueue.php
+++ b/app/Models/ApplicationDeploymentQueue.php
@@ -39,6 +39,7 @@ use OpenApi\Attributes as OA;
'deployment_url' => ['type' => 'string'],
'destination_id' => ['type' => 'string'],
'only_this_server' => ['type' => 'boolean'],
+ 'parent_deployment_uuid' => ['type' => 'string', 'nullable' => true],
'rollback' => ['type' => 'boolean'],
'commit_message' => ['type' => 'string'],
],
@@ -105,6 +106,7 @@ class ApplicationDeploymentQueue extends Model
'deployment_url',
'destination_id',
'only_this_server',
+ 'parent_deployment_uuid',
'rollback',
'commit_message',
'is_api',
diff --git a/app/Models/LocalFileVolume.php b/app/Models/LocalFileVolume.php
index 86d9508d80..1e01d4f43f 100644
--- a/app/Models/LocalFileVolume.php
+++ b/app/Models/LocalFileVolume.php
@@ -4,10 +4,12 @@ namespace App\Models;
use App\Events\FileStorageChanged;
use App\Jobs\ServerStorageSaveJob;
+use Closure;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Database\Eloquent\Relations\MorphTo;
+use Illuminate\Support\Collection;
use Symfony\Component\Yaml\Yaml;
class LocalFileVolume extends BaseModel
@@ -236,21 +238,21 @@ class LocalFileVolume extends BaseModel
return (string) $content;
}
- public function deleteStorageOnServer()
+ /**
+ * Without a server, Coolify deletes the file on every server of the resource.
+ */
+ public function deleteStorageOnServer(?Server $server = null)
{
if ($this->is_host_file) {
return;
}
+ if (is_null($server)) {
+ return $this->runOnEveryServer(fn (Server $server) => $this->deleteStorageOnServer($server));
+ }
$this->load(['service']);
$isService = data_get($this->resource, 'service');
- if ($isService) {
- $workdir = $this->resource->service->workdir();
- $server = $this->resource->service->server;
- } else {
- $workdir = $this->resource->workdir();
- $server = $this->resource->destination->server;
- }
+ $workdir = $isService ? $this->resource->service->workdir() : $this->resource->workdir();
$commands = collect([]);
$path = data_get_str($this, 'fs_path');
if ($path->startsWith('.')) {
@@ -282,21 +284,21 @@ class LocalFileVolume extends BaseModel
}
}
- public function saveStorageOnServer()
+ /**
+ * Without a server, Coolify writes the file on every server of the resource.
+ */
+ public function saveStorageOnServer(?Server $server = null)
{
if ($this->is_host_file) {
return;
}
+ if (is_null($server)) {
+ return $this->runOnEveryServer(fn (Server $server) => $this->saveStorageOnServer($server));
+ }
$this->load(['service']);
$isService = data_get($this->resource, 'service');
- if ($isService) {
- $workdir = $this->resource->service->workdir();
- $server = $this->resource->service->server;
- } else {
- $workdir = $this->resource->workdir();
- $server = $this->resource->destination->server;
- }
+ $workdir = $isService ? $this->resource->service->workdir() : $this->resource->workdir();
$commands = collect([]);
$escapedWorkdir = escapeshellarg($workdir);
@@ -555,6 +557,49 @@ class LocalFileVolume extends BaseModel
: $owner->workdir();
}
+ /**
+ * The main server of the resource, and the additional servers of an application.
+ *
+ * @return Collection<int, Server>
+ */
+ public function servers(): Collection
+ {
+ $this->load(['service']);
+ if (data_get($this->resource, 'service')) {
+ return collect([$this->resource->service->server]);
+ }
+
+ $servers = collect([$this->resource->destination->server]);
+ if ($this->resource instanceof Application) {
+ $servers = $servers->merge($this->resource->additional_servers);
+ }
+
+ return $servers->filter()->unique('id')->values();
+ }
+
+ /**
+ * Runs the callback on every server, also when one server fails, and then throws the first error.
+ *
+ * @param Closure(Server): mixed $callback
+ */
+ protected function runOnEveryServer(Closure $callback): mixed
+ {
+ $result = null;
+ $firstError = null;
+ foreach ($this->servers() as $server) {
+ try {
+ $result = $callback($server);
+ } catch (\Throwable $e) {
+ $firstError ??= $e;
+ }
+ }
+ if ($firstError) {
+ throw $firstError;
+ }
+
+ return $result;
+ }
+
/**
* The Application, Service or standalone database that owns the storage directory.
*/
diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php
index 2ffb959058..fd5ac7731a 100644
--- a/bootstrap/helpers/applications.php
+++ b/bootstrap/helpers/applications.php
@@ -12,7 +12,7 @@ use App\Models\StandaloneDocker;
use Illuminate\Support\Facades\DB;
use Spatie\Url\Url;
-function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null)
+function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null, ?string $parent_deployment_uuid = null)
{
$commit = $commit ?: ($application->git_commit_sha ?: 'HEAD');
$commit = validateGitRef($commit, 'deployment commit');
@@ -31,7 +31,7 @@ function queue_application_deployment(Application $application, string $deployme
$destination_id = $destination->id;
}
- $admission = DB::transaction(function () use ($application, $application_id, $commit, $deployment_uuid, $deployment_url, $destination_id, $docker_registry_image_tag, $force_rebuild, $git_type, $is_api, $is_webhook, $no_questions_asked, $only_this_server, $pull_request_id, $restart_only, $rollback, $server_id, $server_name) {
+ $admission = DB::transaction(function () use ($application, $application_id, $commit, $deployment_uuid, $deployment_url, $destination_id, $docker_registry_image_tag, $force_rebuild, $git_type, $is_api, $is_webhook, $no_questions_asked, $only_this_server, $parent_deployment_uuid, $pull_request_id, $restart_only, $rollback, $server_id, $server_name) {
// Lock stable rows because an empty deployment queue has no row to lock.
Application::query()->whereKey($application_id)->lockForUpdate()->firstOrFail();
$serverForQueueCheck = Server::query()->whereKey($server_id)->lockForUpdate()->firstOrFail();
@@ -81,6 +81,7 @@ function queue_application_deployment(Application $application, string $deployme
'rollback' => $rollback,
'git_type' => $git_type,
'only_this_server' => $only_this_server,
+ 'parent_deployment_uuid' => $parent_deployment_uuid,
]);
});
diff --git a/database/migrations/2026_09_28_121242_add_parent_deployment_uuid_to_application_deployment_queues_table.php b/database/migrations/2026_09_28_121242_add_parent_deployment_uuid_to_application_deployment_queues_table.php
new file mode 100644
index 0000000000..56bef70ce1
--- /dev/null
+++ b/database/migrations/2026_09_28_121242_add_parent_deployment_uuid_to_application_deployment_queues_table.php
@@ -0,0 +1,27 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ /**
+ * Links the deployments on additional servers to the main deployment, so Coolify can send one
+ * notification after all servers finish.
+ */
+ public function up(): void
+ {
+ Schema::table('application_deployment_queues', function (Blueprint $table) {
+ $table->string('parent_deployment_uuid')->nullable()->index();
+ });
+ }
+
+ public function down(): void
+ {
+ Schema::table('application_deployment_queues', function (Blueprint $table) {
+ $table->dropIndex(['parent_deployment_uuid']);
+ $table->dropColumn('parent_deployment_uuid');
+ });
+ }
+};
diff --git a/openapi.json b/openapi.json
index 80646a9bfd..f0d628691b 100644
--- a/openapi.json
+++ b/openapi.json
@@ -23241,6 +23241,10 @@
"only_this_server": {
"type": "boolean"
},
+ "parent_deployment_uuid": {
+ "type": "string",
+ "nullable": true
+ },
"rollback": {
"type": "boolean"
},
diff --git a/openapi.yaml b/openapi.yaml
index 38031f9062..3801868b4d 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -14873,6 +14873,9 @@ components:
type: string
only_this_server:
type: boolean
+ parent_deployment_uuid:
+ type: string
+ nullable: true
rollback:
type: boolean
commit_message:
diff --git a/resources/views/livewire/project/service/storage.blade.php b/resources/views/livewire/project/service/storage.blade.php
index c4b5a35f02..ab15ccd621 100644
--- a/resources/views/livewire/project/service/storage.blade.php
+++ b/resources/views/livewire/project/service/storage.blade.php
@@ -112,6 +112,11 @@
})
}
})">
+ @if ($this->volumeUnavailableReason)
+ <x-callout type="warning" title="Volume mounts are unavailable" class="w-full">
+ {{ $this->volumeUnavailableReason }}
+ </x-callout>
+ @else
<form class="flex w-full flex-col gap-4"
wire:submit='submitPersistentVolume'>
<p class="text-[13px] leading-5 text-neutral-500 dark:text-fg-dim">
@@ -130,6 +135,7 @@
</div>
</div>
</form>
+ @endif
</div>
</div>
</div>
diff --git a/tests/Feature/Api/LifecycleApisTest.php b/tests/Feature/Api/LifecycleApisTest.php
index 24fa127219..95004550b8 100644
--- a/tests/Feature/Api/LifecycleApisTest.php
+++ b/tests/Feature/Api/LifecycleApisTest.php
@@ -482,6 +482,70 @@ describe('Application multi-destination cross-team', function () {
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
});
+ test('attaches a destination on another server of the same team', function () {
+ $secondServer = Server::factory()->create(['team_id' => $this->team->id]);
+ $secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
+
+ $this->withHeaders($this->headers)
+ ->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
+ 'destination_uuid' => $secondDestination->uuid,
+ ])
+ ->assertSuccessful();
+
+ expect($this->application->fresh()->additional_networks)->toHaveCount(1);
+ });
+
+ test('rejects an additional server for an application with persistent storage', function () {
+ LocalPersistentVolume::create([
+ 'name' => 'app-data-'.$this->application->uuid,
+ 'mount_path' => '/data',
+ 'resource_id' => $this->application->id,
+ 'resource_type' => $this->application->getMorphClass(),
+ ]);
+ $secondServer = Server::factory()->create(['team_id' => $this->team->id]);
+ $secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
+
+ $this->withHeaders($this->headers)
+ ->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
+ 'destination_uuid' => $secondDestination->uuid,
+ ])
+ ->assertUnprocessable()
+ ->assertJsonPath('message', 'Applications with persistent storage cannot use multiple servers because volumes are not shared between servers.');
+
+ expect($this->application->fresh()->additional_networks)->toHaveCount(0);
+ });
+
+ test('rejects an additional server for a Docker Compose application', function () {
+ $this->application->update(['build_pack' => 'dockercompose']);
+ $secondServer = Server::factory()->create(['team_id' => $this->team->id]);
+ $secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
+
+ $this->withHeaders($this->headers)
+ ->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
+ 'destination_uuid' => $secondDestination->uuid,
+ ])
+ ->assertUnprocessable();
+
+ expect($this->application->fresh()->additional_networks)->toHaveCount(0);
+ });
+
+ test('rejects a volume for an application with an additional server', function () {
+ $secondServer = Server::factory()->create(['team_id' => $this->team->id]);
+ $secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
+ $this->application->additional_networks()->attach($secondDestination->id, ['server_id' => $secondServer->id]);
+
+ $this->withHeaders($this->headers)
+ ->postJson("/api/v1/applications/{$this->application->uuid}/storages", [
+ 'type' => 'persistent',
+ 'name' => 'data',
+ 'mount_path' => '/data',
+ ])
+ ->assertUnprocessable()
+ ->assertJsonPath('message', 'Applications that use multiple servers cannot have persistent storage because volumes are not shared between servers.');
+
+ expect($this->application->persistentStorages()->count())->toBe(0);
+ });
+
test('lists primary destination', function () {
$response = $this->withHeaders($this->headers)
->getJson("/api/v1/applications/{$this->application->uuid}/destinations");
diff --git a/tests/Feature/ContentFilesBeforeComposeUpTest.php b/tests/Feature/ContentFilesBeforeComposeUpTest.php
index f310026e52..b812ed68f8 100644
--- a/tests/Feature/ContentFilesBeforeComposeUpTest.php
+++ b/tests/Feature/ContentFilesBeforeComposeUpTest.php
@@ -200,8 +200,8 @@ function runComposeStart(object $test, array $properties = []): ContentFilesUpDe
foreach ([
'application' => $test->application->fresh(),
'application_deployment_queue' => $queue,
- 'server' => $test->server,
- 'mainServer' => $test->server,
+ 'server' => $test->server->fresh(),
+ 'mainServer' => $test->server->fresh(),
'deployment_uuid' => 'deployment-uuid',
'workdir' => '/artifacts/deployment-uuid',
'configuration_dir' => $test->application->workdir(),
@@ -343,18 +343,22 @@ test('a preserve-repository deployment writes content files before docker compos
->and($write)->toBeLessThan(ContentFilesUpRecorder::indexOf('deployment', ' up -d'));
});
-test('a deployment to another server does not check the content files of the main server', function () {
+test('a deployment to an additional server writes missing content files on that server', function () {
$volume = contentFilesVolume($this->application, 'config/app.conf');
$otherServer = Server::factory()->create([
'team_id' => $this->team->id,
'private_key_id' => $this->server->private_key_id,
- ]);
+ ])->fresh();
fakeContentFilesServer([$volume->fs_path => 'missing']);
runComposeStart($this, ['server' => $otherServer, 'mainServer' => $otherServer]);
- expect(contentFilesStateChecks())->toBe([]);
- contentFilesAssertNoWrite();
+ $write = ContentFilesUpRecorder::indexOf('ssh', 'base64 -d | tee '.escapeshellarg($volume->fs_path));
+ expect(contentFilesStateChecks())->toHaveCount(1)
+ ->and(contentFilesStateChecks()[0])->toContain("@'{$otherServer->ip}'")
+ ->and($write)->not->toBeNull()
+ ->and(ContentFilesUpRecorder::ssh()[$write])->toContain("@'{$otherServer->ip}'")
+ ->and(ContentFilesUpRecorder::ssh()[$write])->not->toContain("@'{$this->server->ip}'");
});
test('the content file check and write are safe for non-root servers', function () {
diff --git a/tests/Feature/MultiServerApplicationTest.php b/tests/Feature/MultiServerApplicationTest.php
new file mode 100644
index 0000000000..fe6eef7b9e
--- /dev/null
+++ b/tests/Feature/MultiServerApplicationTest.php
@@ -0,0 +1,415 @@
+<?php
+
+use App\Actions\Application\StopApplication;
+use App\Enums\ApplicationDeploymentStatus;
+use App\Exceptions\DeploymentException;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Livewire\Project\Database\Sqlite\ConnectApplication;
+use App\Livewire\Project\Service\Storage;
+use App\Livewire\Project\Shared\Destination;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalFileVolume;
+use App\Models\LocalPersistentVolume;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneSqlite;
+use App\Models\Team;
+use App\Models\User;
+use App\Notifications\Application\DeploymentFailed;
+use App\Notifications\Application\DeploymentSuccess;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Queue;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Queue::fake();
+ Notification::fake();
+ config()->set('cache.default', 'array');
+
+ InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
+
+ $this->team = Team::factory()->create();
+ $this->team->emailNotificationSettings->update([
+ 'smtp_enabled' => true,
+ 'deployment_success_email_notifications' => true,
+ 'deployment_failure_email_notifications' => true,
+ ]);
+ $this->user = User::factory()->create();
+ $this->user->teams()->attach($this->team, ['role' => 'owner']);
+
+ $this->mainServer = Server::factory()->create(['team_id' => $this->team->id, 'name' => 'main-server']);
+ $this->mainDestination = StandaloneDocker::where('server_id', $this->mainServer->id)->firstOrFail();
+ $this->extraServer = Server::factory()->create(['team_id' => $this->team->id, 'name' => 'extra-server']);
+ $this->extraDestination = StandaloneDocker::where('server_id', $this->extraServer->id)->firstOrFail();
+
+ $project = Project::factory()->create(['team_id' => $this->team->id]);
+ $environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]);
+
+ $this->application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $this->mainDestination->id,
+ 'destination_type' => $this->mainDestination->getMorphClass(),
+ 'build_pack' => 'dockerfile',
+ ]);
+ $this->application->additional_networks()->attach($this->extraDestination->id, ['server_id' => $this->extraServer->id]);
+ $this->application->refresh();
+});
+
+/**
+ * @param array<string, mixed> $properties
+ */
+function makeMultiServerDeploymentJob(array $properties): array
+{
+ $reflection = new ReflectionClass(ApplicationDeploymentJob::class);
+ $job = $reflection->newInstanceWithoutConstructor();
+ foreach (['pull_request_id' => 0, 'preview' => null, 'only_this_server' => false, 'commit' => 'HEAD', 'rollback' => false, ...$properties] as $name => $value) {
+ $reflection->getProperty($name)->setValue($job, $value);
+ }
+
+ return [$job, $reflection];
+}
+
+function callMultiServerDeploymentJob(array $jobAndReflection, string $method, mixed ...$arguments): mixed
+{
+ [$job, $reflection] = $jobAndReflection;
+
+ return $reflection->getMethod($method)->invoke($job, ...$arguments);
+}
+
+function createMultiServerDeployment(Application $application, Server $server, string $status, ?string $parentDeploymentUuid = null): ApplicationDeploymentQueue
+{
+ return ApplicationDeploymentQueue::create([
+ 'application_id' => $application->id,
+ 'deployment_uuid' => new_public_id(),
+ 'server_id' => $server->id,
+ 'status' => $status,
+ 'parent_deployment_uuid' => $parentDeploymentUuid,
+ ]);
+}
+
+describe('registry image guard', function () {
+ test('fails a multi-server deployment without a registry image', function () {
+ $job = makeMultiServerDeploymentJob(['application' => $this->application]);
+
+ expect(fn () => callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers'))
+ ->toThrow(DeploymentException::class, 'Before deploying to multiple servers');
+ });
+
+ test('allows a multi-server deployment with a registry image', function () {
+ $this->application->update(['docker_registry_image_name' => 'registry.example.com/app']);
+ $job = makeMultiServerDeploymentJob(['application' => $this->application->fresh()]);
+
+ callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
+ })->throwsNoExceptions();
+
+ test('allows preview deployments, which run only on the main server', function () {
+ $job = makeMultiServerDeploymentJob(['application' => $this->application, 'pull_request_id' => 7]);
+
+ callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
+ })->throwsNoExceptions();
+});
+
+function makeVolumeWarningJob(Application $application, array &$logEntries): array
+{
+ $queue = Mockery::mock(ApplicationDeploymentQueue::class);
+ $queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message, string $type = 'stdout') use (&$logEntries) {
+ $logEntries[] = [$message, $type];
+ });
+
+ return makeMultiServerDeploymentJob(['application' => $application, 'application_deployment_queue' => $queue]);
+}
+
+describe('volume warning', function () {
+ test('warns in the deployment log when an application with a volume uses multiple servers', function () {
+ LocalPersistentVolume::create([
+ 'name' => 'app-data-'.$this->application->uuid,
+ 'mount_path' => '/data',
+ 'resource_id' => $this->application->id,
+ 'resource_type' => $this->application->getMorphClass(),
+ ]);
+ $logEntries = [];
+ $job = makeVolumeWarningJob($this->application->fresh(), $logEntries);
+
+ callMultiServerDeploymentJob($job, 'warnAboutVolumesOnMultipleServers');
+
+ expect($logEntries)->toBe([[
+ 'Warning: This application uses multiple servers and has persistent storage. Volumes are not shared between servers, so each server has its own data.',
+ 'stderr',
+ ]]);
+ });
+
+ test('does not warn when the application has no volume', function () {
+ $logEntries = [];
+ $job = makeVolumeWarningJob($this->application, $logEntries);
+
+ callMultiServerDeploymentJob($job, 'warnAboutVolumesOnMultipleServers');
+
+ expect($logEntries)->toBe([]);
+ });
+});
+
+describe('combined deployment notification', function () {
+ test('links queued additional deployments to the main deployment', function () {
+ $parent = createMultiServerDeployment($this->application, $this->mainServer, ApplicationDeploymentStatus::FINISHED->value);
+ $job = makeMultiServerDeploymentJob([
+ 'application' => $this->application,
+ 'application_deployment_queue' => $parent,
+ 'deployment_uuid' => $parent->deployment_uuid,
+ 'server' => $this->mainServer,
+ 'mainServer' => $this->mainServer,
+ 'destination' => $this->mainDestination,
+ ]);
+
+ expect(callMultiServerDeploymentJob($job, 'deploy_to_additional_destinations'))->toBe(1);
+
+ $child = ApplicationDeploymentQueue::where('server_id', $this->extraServer->id)->sole();
+ expect($child->parent_deployment_uuid)->toBe($parent->deployment_uuid);
+ });
+
+ test('additional servers deploy the same commit and image tag as a rollback on the main server', function () {
+ $parent = ApplicationDeploymentQueue::create([
+ 'application_id' => $this->application->id,
+ 'deployment_uuid' => new_public_id(),
+ 'server_id' => $this->mainServer->id,
+ 'status' => ApplicationDeploymentStatus::FINISHED->value,
+ 'commit' => 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678',
+ 'rollback' => true,
+ 'docker_registry_image_tag' => 'v1.2.3',
+ ]);
+ $job = makeMultiServerDeploymentJob([
+ 'application' => $this->application,
+ 'application_deployment_queue' => $parent,
+ 'deployment_uuid' => $parent->deployment_uuid,
+ 'server' => $this->mainServer,
+ 'mainServer' => $this->mainServer,
+ 'destination' => $this->mainDestination,
+ 'commit' => $parent->commit,
+ 'rollback' => true,
+ ]);
+
+ callMultiServerDeploymentJob($job, 'deploy_to_additional_destinations');
+
+ $child = ApplicationDeploymentQueue::where('server_id', $this->extraServer->id)->sole();
+ expect($child->commit)->toBe('a1b2c3d4e5f60718293a4b5c6d7e8f9012345678')
+ ->and((bool) $child->rollback)->toBeTrue()
+ ->and($child->docker_registry_image_tag)->toBe('v1.2.3');
+ });
+
+ test('does not notify while an additional server is still deploying', function () {
+ $parentUuid = new_public_id();
+ Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
+ $failed = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FAILED->value, $parentUuid);
+ createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::IN_PROGRESS->value, $parentUuid);
+ $job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $failed]);
+
+ callMultiServerDeploymentJob($job, 'handleFailedDeployment');
+
+ Notification::assertNothingSent();
+ });
+
+ test('does not notify before the main deployment queued all additional servers', function () {
+ $parentUuid = new_public_id();
+ $finished = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
+ $job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $finished]);
+
+ callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
+
+ Notification::assertNothingSent();
+ });
+
+ test('sends one success notification after all additional servers succeed', function () {
+ $parentUuid = new_public_id();
+ Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
+ $first = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
+ createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
+ $job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $first]);
+
+ callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
+ callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
+
+ Notification::assertSentToTimes($this->team, DeploymentSuccess::class, 1);
+ Notification::assertSentTo($this->team, DeploymentSuccess::class, fn (DeploymentSuccess $notification) => $notification->deployment_uuid === $parentUuid);
+ Notification::assertNotSentTo($this->team, DeploymentFailed::class);
+ });
+
+ test('sends a failed notification that links to the failed server deployment', function () {
+ $parentUuid = new_public_id();
+ Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
+ $finished = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
+ $failed = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FAILED->value, $parentUuid);
+ $job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $finished]);
+
+ callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
+
+ Notification::assertSentTo($this->team, DeploymentFailed::class, fn (DeploymentFailed $notification) => $notification->deployment_uuid === $failed->deployment_uuid);
+ Notification::assertNotSentTo($this->team, DeploymentSuccess::class);
+ });
+});
+
+/**
+ * Records the deployment commands instead of running them on a server.
+ */
+class MultiServerRecordingDeploymentJob extends ApplicationDeploymentJob
+{
+ /** @var list<string> */
+ public array $recordedCommands = [];
+
+ public function __construct() {}
+
+ public function execute_remote_command(...$commands): void
+ {
+ foreach ($commands as $command) {
+ $this->recordedCommands[] = $command['command'] ?? $command[0];
+ }
+ }
+}
+
+function makeInlineDockerfileJob(Application $application, bool $isAdditionalServer): array
+{
+ $reflection = new ReflectionClass(ApplicationDeploymentJob::class);
+ $job = new MultiServerRecordingDeploymentJob;
+ $queue = Mockery::mock(ApplicationDeploymentQueue::class);
+ $queue->shouldReceive('addLogEntry');
+ foreach ([
+ 'application' => $application,
+ 'application_deployment_queue' => $queue,
+ 'is_this_additional_server' => $isAdditionalServer,
+ 'pull_request_id' => 0,
+ ] as $name => $value) {
+ $reflection->getProperty($name)->setValue($job, $value);
+ }
+ $reflection->getMethod('generate_image_names')->invoke($job);
+
+ return [$job, $reflection];
+}
+
+describe('inline Dockerfile image on additional servers', function () {
+ beforeEach(function () {
+ $this->application->update([
+ 'dockerfile' => "FROM nginx:alpine\n",
+ 'docker_registry_image_name' => 'registry.example.com/nginx-multi',
+ ]);
+ });
+
+ test('an additional server pulls the image that the main server pushed', function () {
+ [$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), true);
+
+ expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeTrue()
+ ->and($job->recordedCommands)->toBe(["docker pull 'registry.example.com/nginx-multi:latest'"]);
+ });
+
+ test('the main server builds the image', function () {
+ [$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), false);
+
+ expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeFalse()
+ ->and($job->recordedCommands)->toBe([]);
+ });
+});
+
+test('stopping continues on the other servers when one server is not functional', function () {
+ $this->mainServer->settings->update(['is_reachable' => false]);
+ $this->extraServer->settings->update(['is_reachable' => false]);
+
+ $result = StopApplication::run($this->application->fresh());
+
+ expect($result)->toContain('main-server')->toContain('extra-server');
+});
+
+test('file storages of an application belong to all of its servers', function () {
+ $file = LocalFileVolume::create([
+ 'fs_path' => '/data/coolify/applications/'.$this->application->uuid.'/config.yml',
+ 'mount_path' => '/app/config.yml',
+ 'content' => 'key: value',
+ 'resource_id' => $this->application->id,
+ 'resource_type' => $this->application->getMorphClass(),
+ ]);
+
+ expect($file->servers()->pluck('id')->all())->toBe([$this->mainServer->id, $this->extraServer->id]);
+});
+
+describe('adding a server in the UI', function () {
+ beforeEach(function () {
+ $this->application->additional_networks()->detach();
+ $this->actingAs($this->user);
+ session(['currentTeam' => $this->team]);
+ });
+
+ test('rejects an application with persistent storage', function () {
+ LocalPersistentVolume::create([
+ 'name' => 'app-data-'.$this->application->uuid,
+ 'mount_path' => '/data',
+ 'resource_id' => $this->application->id,
+ 'resource_type' => $this->application->getMorphClass(),
+ ]);
+
+ Livewire::test(Destination::class, ['resource' => $this->application->fresh()])
+ ->call('addServer', $this->extraDestination->id, $this->extraServer->id)
+ ->assertDispatched('error');
+
+ expect($this->application->fresh()->additional_networks)->toHaveCount(0);
+ });
+
+ test('adds a server to an application without persistent storage', function () {
+ Livewire::test(Destination::class, ['resource' => $this->application->fresh()])
+ ->call('addServer', $this->extraDestination->id, $this->extraServer->id);
+
+ expect($this->application->fresh()->additional_networks)->toHaveCount(1);
+ });
+});
+
+describe('adding a volume to an application with additional servers', function () {
+ beforeEach(function () {
+ $this->actingAs($this->user);
+ session(['currentTeam' => $this->team]);
+ });
+
+ test('the storage page rejects the volume', function () {
+ Livewire::test(Storage::class, ['resource' => $this->application->fresh()])
+ ->assertSee('Volume mounts are unavailable')
+ ->set('name', 'data')
+ ->set('mount_path', '/data')
+ ->call('submitPersistentVolume')
+ ->assertDispatched('error');
+
+ expect($this->application->persistentStorages()->count())->toBe(0);
+ });
+
+ test('connecting a SQLite database rejects the volume', function () {
+ $sqlite = StandaloneSqlite::create([
+ 'name' => 'app-sqlite',
+ 'environment_id' => $this->application->environment_id,
+ 'destination_id' => $this->mainDestination->id,
+ 'destination_type' => $this->mainDestination->getMorphClass(),
+ ]);
+
+ Livewire::test(ConnectApplication::class, ['database' => $sqlite])
+ ->set('applicationUuid', $this->application->uuid)
+ ->set('mountPath', '/app/database')
+ ->call('connect')
+ ->assertNoRedirect();
+
+ expect($this->application->persistentStorages()->count())->toBe(0);
+ });
+
+ test('an application on one server can still add a volume', function () {
+ $this->application->additional_networks()->detach();
+
+ Livewire::test(Storage::class, ['resource' => $this->application->fresh()])
+ ->set('name', 'data')
+ ->set('mount_path', '/data')
+ ->call('submitPersistentVolume')
+ ->assertNotDispatched('error')
+ ->assertDontSee('Volume mounts are unavailable');
+
+ expect($this->application->persistentStorages()->count())->toBe(1);
+ });
+});