An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(servers): isolate cloud status checks from SSH checks
Andras Bacsaic8a332a3bc93
diff --git a/app/Actions/Server/DeleteServer.php b/app/Actions/Server/DeleteServer.phpindex aab889479d..c6f0320133 100644--- a/app/Actions/Server/DeleteServer.php+++ b/app/Actions/Server/DeleteServer.php@@ -122,12 +122,7 @@ class DeleteServer}if (! $token) {- logger()->debug('No Vultr token found for team, skipping Vultr deletion', [- 'team_id' => $teamId,- 'vultr_instance_id' => $vultrInstanceId,- ]);-- return;+ throw new \RuntimeException('No Vultr token found for the server team.');}$vultrService = new VultrService($token->token);@@ -143,6 +138,8 @@ class DeleteServer'vultr_instance_id' => $vultrInstanceId,'team_id' => $teamId,]);++ throw $e;}}@@ -165,12 +162,7 @@ class DeleteServer}if (! $token) {- logger()->debug('No DigitalOcean token found for team, skipping droplet deletion', [- 'team_id' => $teamId,- 'digitalocean_droplet_id' => $digitalOceanDropletId,- ]);-- return;+ throw new \RuntimeException('No DigitalOcean token found for the server team.');}$digitalOceanService = new DigitalOceanService($token->token);@@ -186,6 +178,8 @@ class DeleteServer'digitalocean_droplet_id' => $digitalOceanDropletId,'team_id' => $teamId,]);++ throw $e;}}}diff --git a/app/Http/Controllers/Api/DigitalOceanController.php b/app/Http/Controllers/Api/DigitalOceanController.phpindex fc51e7c013..5bd9d2392e 100644--- a/app/Http/Controllers/Api/DigitalOceanController.php+++ b/app/Http/Controllers/Api/DigitalOceanController.php@@ -15,6 +15,7 @@ use App\Rules\ValidHostname;use App\Services\DigitalOceanService;use Illuminate\Http\JsonResponse;use Illuminate\Http\Request;+use Illuminate\Support\Facades\DB;use OpenApi\Attributes as OA;class DigitalOceanController extends Controller@@ -283,6 +284,10 @@ class DigitalOceanController extends Controllerreturn response()->json(['message' => 'Private key not found.'], 404);}+ $digitalOceanService = null;+ $dropletId = null;+ $server = null;+try {$digitalOceanService = new DigitalOceanService($token->token);$sshKeyId = $this->getOrCreateSshKey($digitalOceanService, $privateKey);@@ -309,29 +314,41 @@ class DigitalOceanController extends Controller$droplet = $digitalOceanService->createDroplet($params);$dropletId = (int) $droplet['id'];- $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);- $ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);- if (! $ipAddress) {- throw new \Exception('No public IP address available for the new droplet.');+ $server = DB::transaction(function () use ($normalizedServerName, $teamId, $privateKey, $token, $dropletId, $droplet): Server {+ $server = Server::create([+ 'name' => $normalizedServerName,+ 'ip' => Server::PLACEHOLDER_IP,+ 'user' => 'root',+ 'port' => 22,+ 'team_id' => $teamId,+ 'private_key_id' => $privateKey->id,+ 'cloud_provider_token_id' => $token->id,+ 'digitalocean_droplet_id' => $dropletId,+ 'digitalocean_droplet_status' => $droplet['status'] ?? null,+ ]);++ $server->proxy->set('status', 'exited');+ $server->proxy->set('type', ProxyTypes::TRAEFIK->value);+ $server->save();++ return $server;+ });++ try {+ $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);+ $ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);++ if ($ipAddress) {+ $server->update([+ 'ip' => $ipAddress,+ 'digitalocean_droplet_status' => $droplet['status'] ?? $server->digitalocean_droplet_status,+ ]);+ }+ } catch (\Throwable $e) {+ report($e);}- $server = Server::create([- 'name' => $normalizedServerName,- 'ip' => $ipAddress,- 'user' => 'root',- 'port' => 22,- 'team_id' => $teamId,- 'private_key_id' => $privateKey->id,- 'cloud_provider_token_id' => $token->id,- 'digitalocean_droplet_id' => $dropletId,- 'digitalocean_droplet_status' => $droplet['status'] ?? null,- ]);-- $server->proxy->set('status', 'exited');- $server->proxy->set('type', ProxyTypes::TRAEFIK->value);- $server->save();-if ($request->instant_validate) {ValidateServer::dispatch($server);}@@ -341,15 +358,17 @@ class DigitalOceanController extends Controller'server_uuid' => $server->uuid,'server_name' => $server->name,'digitalocean_droplet_id' => $dropletId,- 'ip' => $ipAddress,+ 'ip' => $server->ip,]);return response()->json(['uuid' => $server->uuid,'digitalocean_droplet_id' => $dropletId,- 'ip' => $ipAddress,+ 'ip' => $server->ip,])->setStatusCode(201);} catch (RateLimitException $e) {+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);+$response = response()->json(['message' => $e->getMessage()], 429);if ($e->retryAfter !== null) {$response->header('Retry-After', $e->retryAfter);@@ -357,6 +376,8 @@ class DigitalOceanController extends Controllerreturn $response;} catch (\Throwable $e) {+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);+logger()->error('Failed to create DigitalOcean server', ['error' => $e->getMessage(),]);@@ -365,6 +386,19 @@ class DigitalOceanController extends Controller}}+ private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void+ {+ if (! $digitalOceanService || ! $dropletId || $server) {+ return;+ }++ try {+ $digitalOceanService->deleteDroplet($dropletId);+ } catch (\Throwable $e) {+ report($e);+ }+ }+private function getOrCreateSshKey(DigitalOceanService $digitalOceanService, PrivateKey $privateKey): int{$md5Fingerprint = PrivateKey::generateMd5Fingerprint($privateKey->private_key);diff --git a/app/Http/Controllers/Api/VultrController.php b/app/Http/Controllers/Api/VultrController.phpindex 7aaba568e1..60b9d0a0aa 100644--- a/app/Http/Controllers/Api/VultrController.php+++ b/app/Http/Controllers/Api/VultrController.php@@ -52,6 +52,8 @@ class VultrController extends Controllerreturn response()->json(['message' => 'Vultr cloud provider token not found.'], 404);}+ $this->authorize('view', $token);+return $token;}@@ -277,6 +279,8 @@ class VultrController extends Controllerreturn response()->json(['message' => 'Vultr cloud provider token not found.'], 404);}+ $this->authorize('view', $token);+$privateKey = PrivateKey::whereTeamId($teamId)->whereUuid($request->private_key_uuid)->first();if (! $privateKey) {return response()->json(['message' => 'Private key not found.'], 404);@@ -313,7 +317,7 @@ class VultrController extends Controller}$vultrInstance = $vultrService->createInstance($params);- $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? '0.0.0.0';+ $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? Server::PLACEHOLDER_IP;$server = Server::create(['name' => $normalizedServerName,diff --git a/app/Jobs/CleanupOrphanedPreviewContainersJob.php b/app/Jobs/CleanupOrphanedPreviewContainersJob.phpindex 5d3bed4573..e74cba5548 100644--- a/app/Jobs/CleanupOrphanedPreviewContainersJob.php+++ b/app/Jobs/CleanupOrphanedPreviewContainersJob.php@@ -12,6 +12,7 @@ use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Collection;use Illuminate\Support\Facades\Log;/**@@ -53,11 +54,13 @@ class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBe/*** Get all functional servers to check for orphaned containers.*/- private function getServersToCheck(): \Illuminate\Support\Collection+ private function getServersToCheck(): Collection{$query = Server::whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)- ->where('ip', '!=', '1.2.3.4');+ ->whereNotNull('ip')+ ->where('ip', '!=', '')+ ->whereNotIn('ip', Server::PLACEHOLDER_IPS);if (isCloud()) {$query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);@@ -99,7 +102,7 @@ class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBe/*** Get all PR containers on a server (containers with pullRequestId > 0).*/- private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection+ private function getPRContainersOnServer(Server $server): Collection{try {$output = instant_remote_process([diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.phpindex e7a21949cf..46bc89d424 100644--- a/app/Jobs/ScheduledJobManager.php+++ b/app/Jobs/ScheduledJobManager.php@@ -457,7 +457,9 @@ class ScheduledJobManager implements ShouldQueueprivate function getServersForCleanupQuery(): Builder{$query = Server::with('settings')- ->where('ip', '!=', '1.2.3.4');+ ->whereNotNull('ip')+ ->where('ip', '!=', '')+ ->whereNotIn('ip', Server::PLACEHOLDER_IPS);if (isCloud()) {$querydiff --git a/app/Jobs/ServerCloudProviderStatusCheckJob.php b/app/Jobs/ServerCloudProviderStatusCheckJob.phpnew file mode 100644index 0000000000..cd2e51df7d--- /dev/null+++ b/app/Jobs/ServerCloudProviderStatusCheckJob.php@@ -0,0 +1,59 @@+<?php++namespace App\Jobs;++use App\Models\Server;+use Illuminate\Bus\Queueable;+use Illuminate\Contracts\Queue\ShouldBeEncrypted;+use Illuminate\Contracts\Queue\ShouldQueue;+use Illuminate\Foundation\Bus\Dispatchable;+use Illuminate\Queue\InteractsWithQueue;+use Illuminate\Queue\Middleware\WithoutOverlapping;+use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Facades\Log;++class ServerCloudProviderStatusCheckJob implements ShouldBeEncrypted, ShouldQueue+{+ use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;++ public $tries = 1;++ public $timeout = 120;++ public function __construct(public Server $server)+ {+ $this->onQueue('high');+ }++ public function middleware(): array+ {+ return [(new WithoutOverlapping('server-cloud-provider-status-'.$this->server->uuid))->expireAfter(130)->dontRelease()];+ }++ public function handle(): void+ {+ try {+ if (! $this->server->cloudProviderToken) {+ return;+ }++ match ($this->server->cloudProviderToken->provider) {+ 'hetzner' => $this->server->hetzner_server_id+ ? $this->server->refreshHetznerState()+ : null,+ 'vultr' => $this->server->vultr_instance_id+ ? $this->server->refreshVultrState()+ : null,+ 'digitalocean' => $this->server->digitalocean_droplet_id+ ? $this->server->refreshDigitalOceanState()+ : null,+ default => null,+ };+ } catch (\Throwable $e) {+ Log::debug('Cloud provider status check failed', [+ 'server_id' => $this->server->id,+ 'error' => $e->getMessage(),+ ]);+ }+ }+}diff --git a/app/Jobs/ServerConnectionCheckJob.php b/app/Jobs/ServerConnectionCheckJob.phpindex 60adf8c180..f86686df7c 100644--- a/app/Jobs/ServerConnectionCheckJob.php+++ b/app/Jobs/ServerConnectionCheckJob.php@@ -6,7 +6,6 @@ use App\Events\ServerReachabilityChanged;use App\Helpers\SshMultiplexingHelper;use App\Models\Server;use App\Services\ConfigurationRepository;-use App\Services\HetznerService;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;@@ -42,8 +41,12 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue$configRepository->disableSshMux();}- public function handle()+ public function handle(): void{+ if ($this->server->hasPlaceholderIp()) {+ return;+ }+$wasReachable = (bool) $this->server->settings->is_reachable;$wasNotified = (bool) $this->server->unreachable_notification_sent;@@ -62,19 +65,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueuereturn;}- // Check Hetzner server status if applicable- if ($this->server->hetzner_server_id && $this->server->cloudProviderToken) {- $this->checkHetznerStatus();- }-- if ($this->server->vultr_instance_id && $this->server->cloudProviderToken) {- $this->checkVultrStatus();- }-- if ($this->server->digitalocean_droplet_id && $this->server->cloudProviderToken) {- $this->checkDigitalOceanStatus();- }-// Temporarily disable mux if requestedif ($this->disableMux) {$this->disableSshMux();@@ -136,17 +126,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueuepublic function failed(?\Throwable $exception): void{if ($exception instanceof TimeoutExceededException) {- $wasReachable = (bool) $this->server->settings->is_reachable;- $wasNotified = (bool) $this->server->unreachable_notification_sent;-- $this->server->settings->update([- 'is_reachable' => false,- 'is_usable' => false,- ]);- $this->server->increment('unreachable_count');-- $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);-// Delete the queue job so it doesn't appear in Horizon's failed list.$this->job?->delete();}@@ -171,63 +150,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue}}- private function checkHetznerStatus(): void- {- $status = null;-- try {- $hetznerService = new HetznerService($this->server->cloudProviderToken->token);- $serverData = $hetznerService->getServer($this->server->hetzner_server_id);- $status = $serverData['status'] ?? null;-- } catch (\Throwable) {- // Silently ignore — server may have been deleted from Hetzner.- }- if ($this->server->hetzner_server_status !== $status) {- $this->server->update(['hetzner_server_status' => $status]);- $this->server->hetzner_server_status = $status;- if ($status === 'off') {- throw new \Exception('Server is powered off');- }- }-- }-- private function checkVultrStatus(): void- {- try {- $status = $this->server->refreshVultrState();- } catch (\Throwable) {- // Silently ignore transient Vultr API errors.-- return;- }-- if (in_array($status, ['stopped', 'suspended', 'deleted'], true)) {- throw new \Exception('Vultr instance is not running');- }- }-- private function checkDigitalOceanStatus(): void- {- try {- $status = $this->server->refreshDigitalOceanState();- } catch (\Throwable $e) {- Log::debug('ServerConnectionCheck: DigitalOcean status check failed', [- 'server_id' => $this->server->id,- 'error' => $e->getMessage(),- ]);-- return;- }-- $this->server->digitalocean_droplet_status = $status;-- if (in_array($status, ['off', 'archive', 'deleted'], true)) {- throw new \Exception('DigitalOcean droplet is not running');- }- }-private function checkConnection(): bool{try {diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.phpindex 9532282cc5..67c222c24d 100644--- a/app/Jobs/ServerManagerJob.php+++ b/app/Jobs/ServerManagerJob.php@@ -55,6 +55,13 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue// Get all servers to process$servers = $this->getServers();+ // Provider state checks run independently so slow APIs cannot block SSH checks.+ $this->dispatchCloudProviderStatusChecks($servers);++ $servers = $servers+ ->reject(fn (Server $server) => $server->hasPlaceholderIp())+ ->values();+// Dispatch ServerConnectionCheck for all servers efficiently$this->dispatchConnectionChecks($servers);@@ -64,24 +71,45 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueueprivate function getServers(): Collection{- $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');+ $allServers = Server::with(['settings', 'cloudProviderToken']);if (isCloud()) {$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();- $own = Team::find(0)->servers()->with('settings')->get();+ $own = Team::find(0)->servers()->with(['settings', 'cloudProviderToken'])->get();- return $servers->merge($own);+ return $servers->merge($own)->unique('id')->values();} else {return $allServers->get();}}+ private function dispatchCloudProviderStatusChecks(Collection $servers): void+ {+ if (! shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime)) {+ return;+ }++ $servers->each(function (Server $server) {+ $hasCloudResource = $server->hetzner_server_id+ || $server->vultr_instance_id+ || $server->digitalocean_droplet_id;++ if ($hasCloudResource && $server->cloudProviderToken) {+ ServerCloudProviderStatusCheckJob::dispatch($server);+ }+ });+ }+private function dispatchConnectionChecks(Collection $servers): void{if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {$servers->each(function (Server $server) {try {+ if ($server->hasPlaceholderIp()) {+ return;+ }+// Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivityif ($server->isSentinelEnabled() && $server->isSentinelLive()) {return;diff --git a/app/Livewire/Server/New/ByDigitalOcean.php b/app/Livewire/Server/New/ByDigitalOcean.phpindex a59f23efb0..cf27f0f081 100644--- a/app/Livewire/Server/New/ByDigitalOcean.php+++ b/app/Livewire/Server/New/ByDigitalOcean.php@@ -14,6 +14,7 @@ use App\Services\DigitalOceanService;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Illuminate\Http\Client\RequestException;use Illuminate\Support\Collection;+use Illuminate\Support\Facades\DB;use Livewire\Attributes\Locked;use Livewire\Component;@@ -449,6 +450,10 @@ class ByDigitalOcean extends Component{$this->validate();+ $digitalOceanService = null;+ $dropletId = null;+ $server = null;+try {$this->authorize('create', Server::class);@@ -468,24 +473,29 @@ class ByDigitalOcean extends Component$digitalOceanService = new DigitalOceanService($this->getDigitalOceanToken());$droplet = $this->createDigitalOceanDroplet($digitalOceanService);+ $dropletId = (int) $droplet['id'];// Persist the server immediately so the droplet is always tracked// in Coolify, even if waiting for the public IP fails below.- $server = Server::create([- 'name' => strtolower(trim($this->server_name)),- 'ip' => Server::PLACEHOLDER_IP,- 'user' => 'root',- 'port' => 22,- 'team_id' => currentTeam()->id,- 'private_key_id' => $this->private_key_id,- 'cloud_provider_token_id' => $this->selected_token_id,- 'digitalocean_droplet_id' => $droplet['id'],- 'digitalocean_droplet_status' => $droplet['status'] ?? null,- ]);+ $server = DB::transaction(function () use ($dropletId, $droplet): Server {+ $server = Server::create([+ 'name' => strtolower(trim($this->server_name)),+ 'ip' => Server::PLACEHOLDER_IP,+ 'user' => 'root',+ 'port' => 22,+ 'team_id' => currentTeam()->id,+ 'private_key_id' => $this->private_key_id,+ 'cloud_provider_token_id' => $this->selected_token_id,+ 'digitalocean_droplet_id' => $dropletId,+ 'digitalocean_droplet_status' => $droplet['status'] ?? null,+ ]);- $server->proxy->set('status', 'exited');- $server->proxy->set('type', ProxyTypes::TRAEFIK->value);- $server->save();+ $server->proxy->set('status', 'exited');+ $server->proxy->set('type', ProxyTypes::TRAEFIK->value);+ $server->save();++ return $server;+ });try {$droplet = $digitalOceanService->waitForPublicIp($droplet, true, $this->enable_ipv6);@@ -510,10 +520,25 @@ class ByDigitalOcean extends Componentreturn redirectRoute($this, 'server.show', [$server->uuid]);} catch (\Throwable $e) {+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);+return handleError($e, $this);}}+ private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void+ {+ if (! $digitalOceanService || ! $dropletId || $server) {+ return;+ }++ try {+ $digitalOceanService->deleteDroplet($dropletId);+ } catch (\Throwable $e) {+ report($e);+ }+ }+public function render(){return view('livewire.server.new.by-digital-ocean');diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.phpindex 83f4f83d93..6cd0043624 100644--- a/app/Livewire/Server/Show.php+++ b/app/Livewire/Server/Show.php@@ -611,6 +611,7 @@ class Show extends Componentpublic function startVultrInstance(){try {+ $this->authorize('update', $this->server);if (! $this->server->vultr_instance_id || ! $this->server->cloudProviderToken) {$this->dispatch('error', 'This server is not associated with a Vultr instance or token.');diff --git a/app/Models/Server.php b/app/Models/Server.phpindex 928f80fed6..d55df0179c 100644--- a/app/Models/Server.php+++ b/app/Models/Server.php@@ -18,6 +18,7 @@ use App\Notifications\Server\Reachable;use App\Notifications\Server\Unreachable;use App\Services\ConfigurationRepository;use App\Services\DigitalOceanService;+use App\Services\HetznerService;use App\Services\VultrService;use App\Support\ValidationPatterns;use App\Traits\ClearsGlobalSearchCache;@@ -119,6 +120,8 @@ class Server extends BaseModel*/public const PLACEHOLDER_IP = '1.2.3.4';+ public const PLACEHOLDER_IPS = [self::PLACEHOLDER_IP, '0.0.0.0', '::'];+public static $batch_counter = 0;/**@@ -317,9 +320,12 @@ class Server extends BaseModelpublic function hasPlaceholderIp(): bool{// Cast: the saving hook stores the ip as a Stringable in memory.- $ip = (string) $this->ip;+ return self::isPlaceholderIp((string) $this->ip);+ }- return blank($ip) || in_array($ip, [self::PLACEHOLDER_IP, '0.0.0.0', '::'], true);+ public static function isPlaceholderIp(?string $ip): bool+ {+ return blank($ip) || in_array($ip, self::PLACEHOLDER_IPS, true);}/**@@ -328,13 +334,66 @@ class Server extends BaseModel*/public function backfillPlaceholderIp(?string $ip): bool{- if ($ip && $this->hasPlaceholderIp()) {- $this->update(['ip' => $ip]);+ if (self::isPlaceholderIp($ip)) {+ return false;+ }- return true;+ $updated = static::query()+ ->whereKey($this->getKey())+ ->where(function (Builder $query): void {+ $query->whereNull('ip')+ ->orWhere('ip', '')+ ->orWhereIn('ip', self::PLACEHOLDER_IPS);+ })+ ->update(['ip' => $ip]);++ if ($updated === 0) {+ return false;}- return false;+ $this->forceFill(['ip' => $ip]);+ $this->syncOriginalAttribute('ip');+ static::flushIdentityMap();++ return true;+ }++ /**+ * Persist provider status without saving a stale in-memory IP value.+ *+ * @param array<string, mixed> $updates+ */+ private function persistProviderState(array $updates): void+ {+ if (empty($updates)) {+ return;+ }++ static::query()->whereKey($this->getKey())->update($updates);+ $this->forceFill($updates);+ $this->syncOriginalAttributes(array_keys($updates));+ static::flushIdentityMap();+ }++ public function refreshHetznerState(): ?string+ {+ if (! $this->hetzner_server_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'hetzner') {+ return $this->hetzner_server_status;+ }++ $hetznerService = new HetznerService($this->cloudProviderToken->token);+ $server = $hetznerService->getServer($this->hetzner_server_id);+ $status = $server['status'] ?? null;+ $assignedIp = data_get($server, 'public_net.ipv4.ip') ?? data_get($server, 'public_net.ipv6.ip');++ $updates = [];+ if ($this->hetzner_server_status !== $status) {+ $updates['hetzner_server_status'] = $status;+ }+ $this->persistProviderState($updates);+ $this->backfillPlaceholderIp($assignedIp);++ return $status;}public function refreshVultrState(): ?string@@ -352,8 +411,7 @@ class Server extends BaseModel}if ($this->vultr_instance_status !== 'deleted') {- $this->update(['vultr_instance_status' => 'deleted']);- $this->forceFill(['vultr_instance_status' => 'deleted']);+ $this->persistProviderState(['vultr_instance_status' => 'deleted']);}return 'deleted';@@ -368,15 +426,8 @@ class Server extends BaseModelif ($this->vultr_instance_status !== $status) {$updates['vultr_instance_status'] = $status;}-- if ($this->hasPlaceholderIp() && $publicIp) {- $updates['ip'] = $publicIp;- }-- if (! empty($updates)) {- $this->update($updates);- $this->forceFill($updates);- }+ $this->persistProviderState($updates);+ $this->backfillPlaceholderIp($publicIp);return $status;}@@ -393,7 +444,7 @@ class Server extends BaseModel$droplet = $digitalOceanService->getDroplet((int) $this->digitalocean_droplet_id);} catch (RequestException $e) {if ($e->response?->status() === 404) {- $this->update(['digitalocean_droplet_status' => 'deleted']);+ $this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);return 'deleted';}@@ -401,7 +452,7 @@ class Server extends BaseModelthrow $e;} catch (\Throwable $e) {if ((int) $e->getCode() === 404) {- $this->update(['digitalocean_droplet_status' => 'deleted']);+ $this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);return 'deleted';}@@ -416,12 +467,8 @@ class Server extends BaseModel$status = $droplet['status'] ?? null;$ip = $digitalOceanService->getPublicIpAddress($droplet);- $updates = ['digitalocean_droplet_status' => $status];- if ($ip && $this->hasPlaceholderIp()) {- $updates['ip'] = $ip;- }-- $this->update($updates);+ $this->persistProviderState(['digitalocean_droplet_status' => $status]);+ $this->backfillPlaceholderIp($ip);return $status;}diff --git a/tests/Feature/DigitalOceanApiTest.php b/tests/Feature/DigitalOceanApiTest.phpindex 04c1e16bea..8268999576 100644--- a/tests/Feature/DigitalOceanApiTest.php+++ b/tests/Feature/DigitalOceanApiTest.php@@ -3,6 +3,7 @@use App\Models\CloudProviderToken;use App\Models\InstanceSettings;use App\Models\PrivateKey;+use App\Models\Server;use App\Models\Team;use App\Models\User;use Illuminate\Foundation\Testing\RefreshDatabase;@@ -133,6 +134,109 @@ describe('POST /api/v1/servers/digitalocean', function () {&& $request['user_data'] === '#cloud-config');});+ test('tracks the droplet with a placeholder IP when waiting for its IP fails', function () {+ Http::fake([+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([+ 'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],+ ], 201),+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([+ 'ssh_keys' => [],+ 'links' => ['pages' => []],+ ], 200),+ 'https://api.digitalocean.com/v2/droplets' => Http::response([+ 'droplet' => [+ 'id' => 987,+ 'name' => 'waiting-for-ip',+ 'status' => 'new',+ 'networks' => ['v4' => [], 'v6' => []],+ ],+ ], 202),+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([+ 'message' => 'temporary provider failure',+ ], 500),+ ]);++ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ 'Content-Type' => 'application/json',+ ])->postJson('/api/v1/servers/digitalocean', [+ 'cloud_provider_token_id' => $this->digitalOceanToken->uuid,+ 'region' => 'nyc1',+ 'size' => 's-1vcpu-1gb',+ 'image' => 'ubuntu-24-04-x64',+ 'name' => 'waiting-for-ip',+ 'private_key_uuid' => $this->privateKey->uuid,+ ]);++ $response->assertCreated();+ $response->assertJsonFragment([+ 'digitalocean_droplet_id' => 987,+ 'ip' => Server::PLACEHOLDER_IP,+ ]);++ $this->assertDatabaseHas('servers', [+ 'name' => 'waiting-for-ip',+ 'ip' => Server::PLACEHOLDER_IP,+ 'team_id' => $this->team->id,+ 'digitalocean_droplet_id' => 987,+ 'digitalocean_droplet_status' => 'new',+ ]);+ });++ test('deletes the droplet when local server persistence fails', function () {+ Http::fake([+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([+ 'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],+ ], 201),+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([+ 'ssh_keys' => [],+ 'links' => ['pages' => []],+ ], 200),+ 'https://api.digitalocean.com/v2/droplets' => Http::response([+ 'droplet' => [+ 'id' => 987,+ 'name' => 'persistence-fails',+ 'status' => 'active',+ 'networks' => [+ 'v4' => [+ ['ip_address' => '203.0.113.10', 'type' => 'public'],+ ],+ ],+ ],+ ], 202),+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response(null, 204),+ ]);++ $eventDispatcher = Server::getEventDispatcher();+ Server::setEventDispatcher(clone $eventDispatcher);+ Server::created(function (): void {+ throw new RuntimeException('local persistence failed');+ });++ try {+ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ 'Content-Type' => 'application/json',+ ])->postJson('/api/v1/servers/digitalocean', [+ 'cloud_provider_token_id' => $this->digitalOceanToken->uuid,+ 'region' => 'nyc1',+ 'size' => 's-1vcpu-1gb',+ 'image' => 'ubuntu-24-04-x64',+ 'name' => 'persistence-fails',+ 'private_key_uuid' => $this->privateKey->uuid,+ ]);+ } finally {+ Server::setEventDispatcher($eventDispatcher);+ }++ $response->assertServerError();+ $this->assertDatabaseMissing('servers', [+ 'digitalocean_droplet_id' => 987,+ ]);+ Http::assertSent(fn ($request) => $request->method() === 'DELETE'+ && $request->url() === 'https://api.digitalocean.com/v2/droplets/987');+ });+test('validates required fields', function () {$response = $this->withHeaders(['Authorization' => 'Bearer '.$this->bearerToken,diff --git a/tests/Feature/DigitalOceanServerCreationTest.php b/tests/Feature/DigitalOceanServerCreationTest.phpindex b42148dfc4..3d81a5dc97 100644--- a/tests/Feature/DigitalOceanServerCreationTest.php+++ b/tests/Feature/DigitalOceanServerCreationTest.php@@ -113,6 +113,54 @@ it('updates the placeholder IP once the droplet reports one', function () {]);});+it('deletes the droplet when local server persistence fails', function () {+ Http::fake([+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([+ 'ssh_key' => ['id' => 123],+ ], 201),+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([+ 'ssh_keys' => [],+ ], 200),+ 'https://api.digitalocean.com/v2/droplets' => Http::response([+ 'droplet' => [+ 'id' => 555,+ 'status' => 'active',+ 'networks' => [+ 'v4' => [+ ['type' => 'public', 'ip_address' => '203.0.113.40'],+ ],+ ],+ ],+ ], 202),+ 'https://api.digitalocean.com/v2/droplets/555' => Http::response(null, 204),+ ]);++ $eventDispatcher = Server::getEventDispatcher();+ Server::setEventDispatcher(clone $eventDispatcher);+ Server::created(function (): void {+ throw new RuntimeException('local persistence failed');+ });++ try {+ Livewire::test(ByDigitalOcean::class, ['selectedTokenUuid' => $this->digitalOceanToken->uuid])+ ->set('server_name', 'persistence-fails')+ ->set('selected_region', 'nyc1')+ ->set('selected_size', 's-1vcpu-1gb')+ ->set('selected_image', 'ubuntu-24-04-x64')+ ->set('private_key_id', $this->privateKey->id)+ ->call('submit')+ ->assertDispatched('error', 'local persistence failed');+ } finally {+ Server::setEventDispatcher($eventDispatcher);+ }++ $this->assertDatabaseMissing('servers', [+ 'digitalocean_droplet_id' => 555,+ ]);+ Http::assertSent(fn ($request) => $request->method() === 'DELETE'+ && $request->url() === 'https://api.digitalocean.com/v2/droplets/555');+});+it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {Livewire::test(ByDigitalOcean::class)->set('current_step', 2)diff --git a/tests/Feature/ServerCloudProviderStatusCheckJobTest.php b/tests/Feature/ServerCloudProviderStatusCheckJobTest.phpnew file mode 100644index 0000000000..10e1a8c9d8--- /dev/null+++ b/tests/Feature/ServerCloudProviderStatusCheckJobTest.php@@ -0,0 +1,149 @@+<?php++use App\Jobs\ServerCloudProviderStatusCheckJob;+use App\Jobs\ServerConnectionCheckJob;+use App\Jobs\ServerManagerJob;+use App\Models\CloudProviderToken;+use App\Models\InstanceSettings;+use App\Models\PrivateKey;+use App\Models\Server;+use App\Models\Team;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Http;+use Illuminate\Support\Facades\Queue;++uses(RefreshDatabase::class);++function createCloudServerForStatusCheckJobTest(string $provider, array $attributes = []): Server+{+ $team = Team::factory()->create();+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);+ $token = CloudProviderToken::create([+ 'team_id' => $team->id,+ 'provider' => $provider,+ 'token' => "test-{$provider}-token",+ 'name' => ucfirst($provider),+ ]);++ return Server::factory()->create(array_merge([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ 'cloud_provider_token_id' => $token->id,+ 'ip' => Server::PLACEHOLDER_IP,+ ], $attributes));+}++beforeEach(function () {+ InstanceSettings::forceCreate([+ 'id' => 0,+ 'instance_timezone' => 'UTC',+ ]);+ Carbon::setTestNow('2026-07-11 12:00:00');+});++afterEach(function () {+ Carbon::setTestNow();+});++it('syncs provider state for a placeholder server without scheduling SSH', function () {+ $server = createCloudServerForStatusCheckJobTest('digitalocean', [+ 'digitalocean_droplet_id' => 987,+ 'digitalocean_droplet_status' => 'new',+ ]);++ Queue::fake();++ (new ServerManagerJob)->handle();++ Queue::assertPushed(ServerCloudProviderStatusCheckJob::class, fn (ServerCloudProviderStatusCheckJob $job) => $job->server->is($server));+ Queue::assertNotPushed(ServerConnectionCheckJob::class);+});++it('backfills a DigitalOcean placeholder without changing reachability', function () {+ $server = createCloudServerForStatusCheckJobTest('digitalocean', [+ 'digitalocean_droplet_id' => 987,+ 'digitalocean_droplet_status' => 'new',+ ]);+ $server->settings->update([+ 'is_reachable' => true,+ 'is_usable' => true,+ ]);++ Http::fake([+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([+ 'droplet' => [+ 'id' => 987,+ 'status' => 'active',+ 'networks' => [+ 'v4' => [+ ['type' => 'public', 'ip_address' => '203.0.113.10'],+ ],+ ],+ ],+ ]),+ ]);++ (new ServerCloudProviderStatusCheckJob($server))->handle();++ $server->refresh();+ expect($server->ip)->toBe('203.0.113.10')+ ->and($server->digitalocean_droplet_status)->toBe('active')+ ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()+ ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();+});++it('backfills a Hetzner placeholder without changing reachability', function () {+ $server = createCloudServerForStatusCheckJobTest('hetzner', [+ 'hetzner_server_id' => 123,+ 'hetzner_server_status' => 'starting',+ ]);+ $server->settings->update([+ 'is_reachable' => true,+ 'is_usable' => true,+ ]);++ Http::fake([+ 'https://api.hetzner.cloud/v1/servers/123' => Http::response([+ 'server' => [+ 'id' => 123,+ 'status' => 'running',+ 'public_net' => [+ 'ipv4' => ['ip' => '198.51.100.20'],+ ],+ ],+ ]),+ ]);++ (new ServerCloudProviderStatusCheckJob($server))->handle();++ $server->refresh();+ expect($server->ip)->toBe('198.51.100.20')+ ->and($server->hetzner_server_status)->toBe('running')+ ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()+ ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();+});++it('only calls the API matching the cloud provider token', function () {+ $server = createCloudServerForStatusCheckJobTest('digitalocean', [+ 'vultr_instance_id' => 'unrelated-vultr-instance',+ 'digitalocean_droplet_id' => 987,+ 'digitalocean_droplet_status' => 'new',+ ]);++ Http::fake([+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([+ 'droplet' => [+ 'id' => 987,+ 'status' => 'active',+ 'networks' => ['v4' => []],+ ],+ ]),+ 'https://api.vultr.com/*' => Http::response(status: 500),+ ]);++ (new ServerCloudProviderStatusCheckJob($server))->handle();++ Http::assertSentCount(1);+ Http::assertSent(fn ($request) => $request->url() === 'https://api.digitalocean.com/v2/droplets/987');+});diff --git a/tests/Feature/ServerConnectionCheckIsolationTest.php b/tests/Feature/ServerConnectionCheckIsolationTest.phpnew file mode 100644index 0000000000..d098c39b55--- /dev/null+++ b/tests/Feature/ServerConnectionCheckIsolationTest.php@@ -0,0 +1,80 @@+<?php++use App\Jobs\ServerConnectionCheckJob;+use App\Models\CloudProviderToken;+use App\Models\PrivateKey;+use App\Models\Server;+use App\Models\Team;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Http;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;++uses(RefreshDatabase::class);++function createServerForConnectionIsolationTest(array $attributes = []): Server+{+ $team = Team::factory()->create();+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);++ return Server::factory()->create(array_merge([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ ], $attributes));+}++beforeEach(function () {+ Storage::fake('ssh-keys');+});++it('never attempts SSH to a placeholder address', function (string $placeholderIp) {+ $server = createServerForConnectionIsolationTest(['ip' => $placeholderIp]);+ $server->settings->update([+ 'is_reachable' => true,+ 'is_usable' => true,+ ]);++ Process::fake();++ (new ServerConnectionCheckJob($server, disableMux: false))->handle();++ Process::assertNothingRan();+ expect($server->fresh()->unreachable_count)->toBe(0)+ ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()+ ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();+})->with([+ 'reserved provisioning address' => Server::PLACEHOLDER_IP,+ 'unspecified IPv4 address' => '0.0.0.0',+ 'unspecified IPv6 address' => '::',+]);++it('does not call cloud provider APIs during an SSH connection check', function () {+ $server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);+ $token = CloudProviderToken::create([+ 'team_id' => $server->team_id,+ 'provider' => 'vultr',+ 'token' => 'test-vultr-token',+ 'name' => 'Vultr',+ ]);+ $server->update([+ 'cloud_provider_token_id' => $token->id,+ 'vultr_instance_id' => 'instance-1',+ 'vultr_instance_status' => 'active',+ ]);++ Http::fake([+ 'https://api.vultr.com/*' => Http::response([+ 'instance' => ['id' => 'instance-1', 'status' => 'active'],+ ]),+ ]);+ Process::fake([+ '*' => Process::result(+ output: '{"Server":{"Version":"27.0.0"}}',+ exitCode: 0,+ ),+ ]);++ (new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();++ Http::assertNothingSent();+});diff --git a/tests/Feature/VultrApiTest.php b/tests/Feature/VultrApiTest.phpindex 8a96ec6857..9eb135baf5 100644--- a/tests/Feature/VultrApiTest.php+++ b/tests/Feature/VultrApiTest.php@@ -1,11 +1,15 @@<?php+use App\Http\Controllers\Api\VultrController;use App\Models\CloudProviderToken;use App\Models\InstanceSettings;use App\Models\PrivateKey;+use App\Models\Server;use App\Models\Team;use App\Models\User;+use Illuminate\Auth\Access\AuthorizationException;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Http\Request;use Illuminate\Support\Facades\Http;uses(RefreshDatabase::class);@@ -165,6 +169,27 @@ describe('GET /api/v1/vultr/ssh-keys', function () {});});+test('member read token cannot use a stored Vultr token for metadata endpoints', function (string $endpoint) {+ $member = User::factory()->create();+ $this->team->members()->attach($member->id, ['role' => 'member']);+ $memberToken = $member->createToken('member-read', ['read'])->plainTextToken;++ Http::fake();++ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$memberToken,+ 'Content-Type' => 'application/json',+ ])->getJson($endpoint.'?cloud_provider_token_uuid='.$this->vultrToken->uuid);++ $response->assertForbidden();+ Http::assertNothingSent();+})->with([+ '/api/v1/vultr/regions',+ '/api/v1/vultr/plans',+ '/api/v1/vultr/os',+ '/api/v1/vultr/ssh-keys',+]);+describe('POST /api/v1/servers/vultr', function () {test('creates a Vultr server', function () {Http::fake([@@ -223,6 +248,11 @@ describe('POST /api/v1/servers/vultr', function () {});test('waits for Vultr to assign a real public IP when creation returns placeholder IP', function () {+ $createdIp = null;+ Server::created(function (Server $server) use (&$createdIp): void {+ $createdIp = (string) $server->ip;+ });+Http::fake(['https://api.vultr.com/v2/ssh-keys' => Http::response(['ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],@@ -262,6 +292,8 @@ describe('POST /api/v1/servers/vultr', function () {$response->assertStatus(201);$response->assertJsonFragment(['ip' => '1.2.3.4']);+ expect($createdIp)->toBe(Server::PLACEHOLDER_IP);+$this->assertDatabaseHas('servers', ['name' => 'test-server','ip' => '1.2.3.4',@@ -269,6 +301,34 @@ describe('POST /api/v1/servers/vultr', function () {]);});+ test('server creation authorizes access to the stored Vultr token', function () {+ $member = User::factory()->create();+ $this->team->members()->attach($member->id, ['role' => 'member']);+ $this->actingAs($member);+ $member->withAccessToken($member->createToken('member-all', ['*'])->accessToken);++ $request = Request::create(+ uri: '/api/v1/servers/vultr',+ method: 'POST',+ server: ['CONTENT_TYPE' => 'application/json'],+ content: json_encode([+ 'cloud_provider_token_uuid' => $this->vultrToken->uuid,+ 'region' => 'ewr',+ 'plan' => 'vc2-1c-1gb',+ 'os_id' => 2284,+ 'name' => 'test-server',+ 'private_key_uuid' => $this->privateKey->uuid,+ ], JSON_THROW_ON_ERROR),+ );+ $request->setUserResolver(fn () => $member);++ Http::fake();++ expect(fn () => app(VultrController::class)->createServer($request))+ ->toThrow(AuthorizationException::class);+ Http::assertNothingSent();+ });+test('reuses existing matching Vultr SSH key', function () {Http::fake(['https://api.vultr.com/v2/ssh-keys*' => Http::response([diff --git a/tests/Feature/VultrServerLifecycleTest.php b/tests/Feature/VultrServerLifecycleTest.phpindex c89ca47b45..cb3b3108c0 100644--- a/tests/Feature/VultrServerLifecycleTest.php+++ b/tests/Feature/VultrServerLifecycleTest.php@@ -123,6 +123,8 @@ it('updates placeholder server IP when Vultr status refresh returns assigned pub});it('does not overwrite an established server IP during Vultr status refresh', function () {+ $this->server->update(['ip' => '198.51.100.20']);+Http::fake(['https://api.vultr.com/v2/instances/instance-1' => Http::response(['instance' => [@@ -135,12 +137,12 @@ it('does not overwrite an established server IP during Vultr status refresh', fuLivewire::test(Show::class, ['server_uuid' => $this->server->uuid])->call('checkVultrInstanceStatus', true)- ->assertSet('ip', '1.2.3.4')+ ->assertSet('ip', '198.51.100.20')->assertSet('vultrInstanceStatus', 'active');$this->assertDatabaseHas('servers', ['id' => $this->server->id,- 'ip' => '1.2.3.4',+ 'ip' => '198.51.100.20','vultr_instance_status' => 'active',]);});@@ -270,6 +272,26 @@ it('starts a Vultr instance', function () {Http::assertSent(fn ($request) => $request->url() === 'https://api.vultr.com/v2/instances/instance-1/start');});+it('prevents members from starting a Vultr instance', function () {+ $this->server->update(['vultr_instance_status' => 'stopped']);++ $member = User::factory()->create();+ $this->team->members()->attach($member->id, ['role' => 'member']);+ $this->actingAs($member);+ session(['currentTeam' => $this->team]);++ Http::fake([+ 'https://api.vultr.com/v2/instances/instance-1/start' => Http::response([], 204),+ ]);++ Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])+ ->call('startVultrInstance')+ ->assertDispatched('error');++ expect($this->server->fresh()->vultr_instance_status)->toBe('stopped');+ Http::assertNothingSent();+});+it('links a server to Vultr by matching IP', function () {$unlinkedServer = Server::factory()->create(['team_id' => $this->team->id,diff --git a/tests/Unit/DigitalOceanDeleteServerTest.php b/tests/Unit/DigitalOceanDeleteServerTest.phpindex 5b0989607e..60cd5ba502 100644--- a/tests/Unit/DigitalOceanDeleteServerTest.php+++ b/tests/Unit/DigitalOceanDeleteServerTest.php@@ -6,6 +6,7 @@ use App\Models\PrivateKey;use App\Models\Server;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Http\Client\RequestException;use Illuminate\Support\Facades\Http;use Tests\TestCase;@@ -44,3 +45,59 @@ it('deletes the DigitalOcean droplet when requested', function () {Http::assertSent(fn ($request) => $request->method() === 'DELETE'&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');});++it('retains the server and surfaces a DigitalOcean deletion failure', function () {+ Http::fake([+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([+ 'message' => 'deletion failed',+ ], 500),+ ]);++ $team = Team::factory()->create();+ $token = CloudProviderToken::factory()->create([+ 'team_id' => $team->id,+ 'provider' => 'digitalocean',+ 'token' => 'test-digitalocean-token',+ ]);+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ 'cloud_provider_token_id' => $token->id,+ 'digitalocean_droplet_id' => 987,+ ]);+ $server->delete();++ expect(fn () => DeleteServer::run(+ serverId: $server->id,+ deleteFromDigitalOcean: true,+ digitalOceanDropletId: 987,+ cloudProviderTokenId: $token->id,+ teamId: $team->id,+ ))->toThrow(RequestException::class, 'status code 500');++ expect(Server::withTrashed()->find($server->id))->not->toBeNull();+});++it('retains the server when no DigitalOcean token can delete the droplet', function () {+ Http::preventStrayRequests();++ $team = Team::factory()->create();+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ 'cloud_provider_token_id' => null,+ 'digitalocean_droplet_id' => 987,+ ]);+ $server->delete();++ expect(fn () => DeleteServer::run(+ serverId: $server->id,+ deleteFromDigitalOcean: true,+ digitalOceanDropletId: 987,+ teamId: $team->id,+ ))->toThrow(RuntimeException::class, 'No DigitalOcean token found');++ expect(Server::withTrashed()->find($server->id))->not->toBeNull();+});diff --git a/tests/Unit/DigitalOceanServerStateTest.php b/tests/Unit/DigitalOceanServerStateTest.phpindex 93b3379bb9..725a38c5bf 100644--- a/tests/Unit/DigitalOceanServerStateTest.php+++ b/tests/Unit/DigitalOceanServerStateTest.php@@ -66,8 +66,8 @@ it('backfills a placeholder IP from the DigitalOcean droplet state', function ()expect($server->fresh()->ip)->toBe('203.0.113.10');});-it('does not overwrite a real IP from the DigitalOcean droplet state', function () {- $server = createDigitalOceanServerForStateTest('active', ['ip' => '198.51.100.20']);+it('does not overwrite an administrator configured address from the DigitalOcean droplet state', function (string $configuredAddress) {+ $server = createDigitalOceanServerForStateTest('active', ['ip' => $configuredAddress]);Http::fake(['https://api.digitalocean.com/v2/droplets/987' => Http::response([@@ -84,7 +84,38 @@ it('does not overwrite a real IP from the DigitalOcean droplet state', function]);$server->refreshDigitalOceanState();- expect($server->fresh()->ip)->toBe('198.51.100.20');+ expect($server->fresh()->ip)->toBe($configuredAddress);+})->with([+ 'public IPv4' => '198.51.100.20',+ 'private IPv4' => '10.10.0.12',+ 'IPv6' => '2001:db8::10',+ 'tunnel hostname' => 'server.internal.example.com',+]);++it('does not overwrite an address configured while DigitalOcean state is loading', function () {+ $server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);++ Http::fake(function () use ($server) {+ Server::query()->findOrFail($server->id)->update([+ 'ip' => 'server.internal.example.com',+ ]);++ return Http::response([+ 'droplet' => [+ 'id' => 987,+ 'status' => 'active',+ 'networks' => [+ 'v4' => [+ ['type' => 'public', 'ip_address' => '203.0.113.10'],+ ],+ ],+ ],+ ], 200);+ });++ expect($server->refreshDigitalOceanState())->toBe('active')+ ->and($server->fresh()->ip)->toBe('server.internal.example.com')+ ->and($server->digitalocean_droplet_status)->toBe('active');});it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {diff --git a/tests/Unit/ServerBackoffTest.php b/tests/Unit/ServerBackoffTest.phpindex 7c9910a5cb..3bbf41b3a2 100644--- a/tests/Unit/ServerBackoffTest.php+++ b/tests/Unit/ServerBackoffTest.php@@ -127,7 +127,7 @@ describe('shouldSkipDueToBackoff', function () {});describe('ServerConnectionCheckJob unreachable_count', function () {- it('marks Vultr servers unreachable when provider status is unavailable', function () {+ it('marks servers unreachable when SSH is unavailable', function () {Event::fake([ServerReachabilityChanged::class]);$settings = Mockery::mock();@@ -141,14 +141,12 @@ describe('ServerConnectionCheckJob unreachable_count', function () {$server->shouldReceive('getAttribute')->andReturnUsing(fn (string $key) => match ($key) {'settings' => $settings,'unreachable_notification_sent' => false,- 'vultr_instance_id' => 'instance-1',- 'cloudProviderToken' => (object) ['token' => 'test-token'],+ 'ip' => '203.0.113.10','id' => 1,'name' => 'test-server','unreachable_count' => 1,default => null,});- $server->shouldReceive('refreshVultrState')->once()->andReturn('stopped');$server->shouldReceive('increment')->with('unreachable_count')->once();$server->id = 1;$server->name = 'test-server';@@ -158,22 +156,20 @@ describe('ServerConnectionCheckJob unreachable_count', function () {$job->handle();});- it('increments unreachable_count on timeout', function () {+ it('does not change reachability on a job-level timeout', function () {Event::fake([ServerReachabilityChanged::class]);$settings = Mockery::mock();$settings->is_reachable = true;- $settings->shouldReceive('update')- ->with(['is_reachable' => false, 'is_usable' => false])- ->once();+ $settings->shouldNotReceive('update');$server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();$server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);$server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);- $server->shouldReceive('increment')->with('unreachable_count')->once();+ $server->shouldNotReceive('increment');$server->id = 1;$server->name = 'test-server';- $server->unreachable_count = 1; // Will become 2 after increment in real code; mock keeps value as-is+ $server->unreachable_count = 1;$job = new ServerConnectionCheckJob($server);$job->failed(new TimeoutExceededException);diff --git a/tests/Unit/ServerPlaceholderIpTest.php b/tests/Unit/ServerPlaceholderIpTest.phpindex b471ae7e13..94787b4d47 100644--- a/tests/Unit/ServerPlaceholderIpTest.php+++ b/tests/Unit/ServerPlaceholderIpTest.php@@ -1,5 +1,7 @@<?php+use App\Jobs\CleanupOrphanedPreviewContainersJob;+use App\Jobs\ScheduledJobManager;use App\Models\PrivateKey;use App\Models\Server;use App\Models\Team;@@ -60,8 +62,64 @@ it('ignores a missing IP when backfilling', function () {expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);});+it('does not replace a placeholder with another placeholder', function (string $replacementIp) {+ $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);++ expect($server->backfillPlaceholderIp($replacementIp))->toBeFalse();+ expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);+})->with([+ 'unspecified IPv4 address' => '0.0.0.0',+ 'unspecified IPv6 address' => '::',+]);++it('does not overwrite an address configured after the placeholder model was loaded', function () {+ $staleServer = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);+ $concurrentServer = Server::query()->findOrFail($staleServer->id);++ $concurrentServer->update(['ip' => 'server.internal.example.com']);++ expect($staleServer->backfillPlaceholderIp('203.0.113.10'))->toBeFalse()+ ->and($staleServer->fresh()->ip)->toBe('server.internal.example.com');+});+it('skips servers with a placeholder IP in scheduled jobs', function () {$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);expect($server->skipServer())->toBeTrue();});++it('excludes every placeholder address from scheduled Docker cleanup', function () {+ $realServer = createServerForPlaceholderIpTest('203.0.113.10');++ foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {+ Server::factory()->create([+ 'team_id' => $realServer->team_id,+ 'private_key_id' => $realServer->private_key_id,+ 'ip' => $placeholderIp,+ ]);+ }++ $method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');+ $servers = $method->invoke(new ScheduledJobManager)->get();++ expect($servers->modelKeys())->toBe([$realServer->id]);+});++it('excludes every placeholder address from orphaned preview cleanup', function () {+ $realServer = createServerForPlaceholderIpTest('203.0.113.10');+ $realServer->settings->update(['is_reachable' => true, 'is_usable' => true]);++ foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {+ $server = Server::factory()->create([+ 'team_id' => $realServer->team_id,+ 'private_key_id' => $realServer->private_key_id,+ 'ip' => $placeholderIp,+ ]);+ $server->settings->update(['is_reachable' => true, 'is_usable' => true]);+ }++ $method = new ReflectionMethod(CleanupOrphanedPreviewContainersJob::class, 'getServersToCheck');+ $servers = $method->invoke(new CleanupOrphanedPreviewContainersJob);++ expect($servers->modelKeys())->toBe([$realServer->id]);+});diff --git a/tests/Unit/VultrDeleteServerTest.php b/tests/Unit/VultrDeleteServerTest.phpindex 05891eed7c..82173d40d4 100644--- a/tests/Unit/VultrDeleteServerTest.php+++ b/tests/Unit/VultrDeleteServerTest.php@@ -7,6 +7,7 @@ use App\Models\PrivateKey;use App\Models\Server;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Http\Client\RequestException;use Illuminate\Support\Facades\Http;use Tests\TestCase;@@ -116,3 +117,52 @@ it('does not use another team Vultr token when deleting an instance', function (expect($request->header('Authorization'))->toBe(['Bearer test-vultr-token']);});++it('retains the server and surfaces a Vultr deletion failure', function () {+ Http::fake([+ 'https://api.vultr.com/v2/instances/instance-1' => Http::response([+ 'error' => 'deletion failed',+ ], 500),+ ]);++ $server = Server::factory()->create([+ 'team_id' => $this->team->id,+ 'private_key_id' => $this->privateKey->id,+ 'cloud_provider_token_id' => $this->vultrToken->id,+ 'vultr_instance_id' => 'instance-1',+ ]);+ $server->delete();++ expect(fn () => DeleteServer::run(+ serverId: $server->id,+ cloudProviderTokenId: $this->vultrToken->id,+ teamId: $this->team->id,+ deleteFromVultr: true,+ vultrInstanceId: 'instance-1'+ ))->toThrow(RequestException::class, 'status code 500');++ expect(Server::withTrashed()->find($server->id))->not->toBeNull();+});++it('retains the server when no Vultr token can delete the instance', function () {+ Http::preventStrayRequests();++ $this->vultrToken->delete();++ $server = Server::factory()->create([+ 'team_id' => $this->team->id,+ 'private_key_id' => $this->privateKey->id,+ 'cloud_provider_token_id' => null,+ 'vultr_instance_id' => 'instance-1',+ ]);+ $server->delete();++ expect(fn () => DeleteServer::run(+ serverId: $server->id,+ teamId: $this->team->id,+ deleteFromVultr: true,+ vultrInstanceId: 'instance-1'+ ))->toThrow(RuntimeException::class, 'No Vultr token found');++ expect(Server::withTrashed()->find($server->id))->not->toBeNull();+});