Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(servers): isolate cloud status checks from SSH checks

Andras Bacsaic8a332a3bc93
diff --git a/app/Actions/Server/DeleteServer.php b/app/Actions/Server/DeleteServer.php
index aab889479d..c6f0320133 100644
--- a/app/Actions/Server/DeleteServer.php
+++ b/app/Actions/Server/DeleteServer.php
@@ -122,12 +122,7 @@ class DeleteServer
}
if (! $token) {
- logger()->debug('No Vultr token found for team, skipping Vultr deletion', [
- 'team_id' => $teamId,
- 'vultr_instance_id' => $vultrInstanceId,
- ]);
-
- return;
+ throw new \RuntimeException('No Vultr token found for the server team.');
}
$vultrService = new VultrService($token->token);
@@ -143,6 +138,8 @@ class DeleteServer
'vultr_instance_id' => $vultrInstanceId,
'team_id' => $teamId,
]);
+
+ throw $e;
}
}
@@ -165,12 +162,7 @@ class DeleteServer
}
if (! $token) {
- logger()->debug('No DigitalOcean token found for team, skipping droplet deletion', [
- 'team_id' => $teamId,
- 'digitalocean_droplet_id' => $digitalOceanDropletId,
- ]);
-
- return;
+ throw new \RuntimeException('No DigitalOcean token found for the server team.');
}
$digitalOceanService = new DigitalOceanService($token->token);
@@ -186,6 +178,8 @@ class DeleteServer
'digitalocean_droplet_id' => $digitalOceanDropletId,
'team_id' => $teamId,
]);
+
+ throw $e;
}
}
}
diff --git a/app/Http/Controllers/Api/DigitalOceanController.php b/app/Http/Controllers/Api/DigitalOceanController.php
index fc51e7c013..5bd9d2392e 100644
--- a/app/Http/Controllers/Api/DigitalOceanController.php
+++ b/app/Http/Controllers/Api/DigitalOceanController.php
@@ -15,6 +15,7 @@ use App\Rules\ValidHostname;
use App\Services\DigitalOceanService;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
use OpenApi\Attributes as OA;
class DigitalOceanController extends Controller
@@ -283,6 +284,10 @@ class DigitalOceanController extends Controller
return response()->json(['message' => 'Private key not found.'], 404);
}
+ $digitalOceanService = null;
+ $dropletId = null;
+ $server = null;
+
try {
$digitalOceanService = new DigitalOceanService($token->token);
$sshKeyId = $this->getOrCreateSshKey($digitalOceanService, $privateKey);
@@ -309,29 +314,41 @@ class DigitalOceanController extends Controller
$droplet = $digitalOceanService->createDroplet($params);
$dropletId = (int) $droplet['id'];
- $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);
- $ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);
- if (! $ipAddress) {
- throw new \Exception('No public IP address available for the new droplet.');
+ $server = DB::transaction(function () use ($normalizedServerName, $teamId, $privateKey, $token, $dropletId, $droplet): Server {
+ $server = Server::create([
+ 'name' => $normalizedServerName,
+ 'ip' => Server::PLACEHOLDER_IP,
+ 'user' => 'root',
+ 'port' => 22,
+ 'team_id' => $teamId,
+ 'private_key_id' => $privateKey->id,
+ 'cloud_provider_token_id' => $token->id,
+ 'digitalocean_droplet_id' => $dropletId,
+ 'digitalocean_droplet_status' => $droplet['status'] ?? null,
+ ]);
+
+ $server->proxy->set('status', 'exited');
+ $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
+ $server->save();
+
+ return $server;
+ });
+
+ try {
+ $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);
+ $ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);
+
+ if ($ipAddress) {
+ $server->update([
+ 'ip' => $ipAddress,
+ 'digitalocean_droplet_status' => $droplet['status'] ?? $server->digitalocean_droplet_status,
+ ]);
+ }
+ } catch (\Throwable $e) {
+ report($e);
}
- $server = Server::create([
- 'name' => $normalizedServerName,
- 'ip' => $ipAddress,
- 'user' => 'root',
- 'port' => 22,
- 'team_id' => $teamId,
- 'private_key_id' => $privateKey->id,
- 'cloud_provider_token_id' => $token->id,
- 'digitalocean_droplet_id' => $dropletId,
- 'digitalocean_droplet_status' => $droplet['status'] ?? null,
- ]);
-
- $server->proxy->set('status', 'exited');
- $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
- $server->save();
-
if ($request->instant_validate) {
ValidateServer::dispatch($server);
}
@@ -341,15 +358,17 @@ class DigitalOceanController extends Controller
'server_uuid' => $server->uuid,
'server_name' => $server->name,
'digitalocean_droplet_id' => $dropletId,
- 'ip' => $ipAddress,
+ 'ip' => $server->ip,
]);
return response()->json([
'uuid' => $server->uuid,
'digitalocean_droplet_id' => $dropletId,
- 'ip' => $ipAddress,
+ 'ip' => $server->ip,
])->setStatusCode(201);
} catch (RateLimitException $e) {
+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
+
$response = response()->json(['message' => $e->getMessage()], 429);
if ($e->retryAfter !== null) {
$response->header('Retry-After', $e->retryAfter);
@@ -357,6 +376,8 @@ class DigitalOceanController extends Controller
return $response;
} catch (\Throwable $e) {
+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
+
logger()->error('Failed to create DigitalOcean server', [
'error' => $e->getMessage(),
]);
@@ -365,6 +386,19 @@ class DigitalOceanController extends Controller
}
}
+ private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void
+ {
+ if (! $digitalOceanService || ! $dropletId || $server) {
+ return;
+ }
+
+ try {
+ $digitalOceanService->deleteDroplet($dropletId);
+ } catch (\Throwable $e) {
+ report($e);
+ }
+ }
+
private function getOrCreateSshKey(DigitalOceanService $digitalOceanService, PrivateKey $privateKey): int
{
$md5Fingerprint = PrivateKey::generateMd5Fingerprint($privateKey->private_key);
diff --git a/app/Http/Controllers/Api/VultrController.php b/app/Http/Controllers/Api/VultrController.php
index 7aaba568e1..60b9d0a0aa 100644
--- a/app/Http/Controllers/Api/VultrController.php
+++ b/app/Http/Controllers/Api/VultrController.php
@@ -52,6 +52,8 @@ class VultrController extends Controller
return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
}
+ $this->authorize('view', $token);
+
return $token;
}
@@ -277,6 +279,8 @@ class VultrController extends Controller
return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
}
+ $this->authorize('view', $token);
+
$privateKey = PrivateKey::whereTeamId($teamId)->whereUuid($request->private_key_uuid)->first();
if (! $privateKey) {
return response()->json(['message' => 'Private key not found.'], 404);
@@ -313,7 +317,7 @@ class VultrController extends Controller
}
$vultrInstance = $vultrService->createInstance($params);
- $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? '0.0.0.0';
+ $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? Server::PLACEHOLDER_IP;
$server = Server::create([
'name' => $normalizedServerName,
diff --git a/app/Jobs/CleanupOrphanedPreviewContainersJob.php b/app/Jobs/CleanupOrphanedPreviewContainersJob.php
index 5d3bed4573..e74cba5548 100644
--- a/app/Jobs/CleanupOrphanedPreviewContainersJob.php
+++ b/app/Jobs/CleanupOrphanedPreviewContainersJob.php
@@ -12,6 +12,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
/**
@@ -53,11 +54,13 @@ class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBe
/**
* Get all functional servers to check for orphaned containers.
*/
- private function getServersToCheck(): \Illuminate\Support\Collection
+ private function getServersToCheck(): Collection
{
$query = Server::whereRelation('settings', 'is_usable', true)
->whereRelation('settings', 'is_reachable', true)
- ->where('ip', '!=', '1.2.3.4');
+ ->whereNotNull('ip')
+ ->where('ip', '!=', '')
+ ->whereNotIn('ip', Server::PLACEHOLDER_IPS);
if (isCloud()) {
$query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);
@@ -99,7 +102,7 @@ class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBe
/**
* Get all PR containers on a server (containers with pullRequestId > 0).
*/
- private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection
+ private function getPRContainersOnServer(Server $server): Collection
{
try {
$output = instant_remote_process([
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php
index e7a21949cf..46bc89d424 100644
--- a/app/Jobs/ScheduledJobManager.php
+++ b/app/Jobs/ScheduledJobManager.php
@@ -457,7 +457,9 @@ class ScheduledJobManager implements ShouldQueue
private function getServersForCleanupQuery(): Builder
{
$query = Server::with('settings')
- ->where('ip', '!=', '1.2.3.4');
+ ->whereNotNull('ip')
+ ->where('ip', '!=', '')
+ ->whereNotIn('ip', Server::PLACEHOLDER_IPS);
if (isCloud()) {
$query
diff --git a/app/Jobs/ServerCloudProviderStatusCheckJob.php b/app/Jobs/ServerCloudProviderStatusCheckJob.php
new file mode 100644
index 0000000000..cd2e51df7d
--- /dev/null
+++ b/app/Jobs/ServerCloudProviderStatusCheckJob.php
@@ -0,0 +1,59 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Server;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ServerCloudProviderStatusCheckJob implements ShouldBeEncrypted, ShouldQueue
+{
+ use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+ public $tries = 1;
+
+ public $timeout = 120;
+
+ public function __construct(public Server $server)
+ {
+ $this->onQueue('high');
+ }
+
+ public function middleware(): array
+ {
+ return [(new WithoutOverlapping('server-cloud-provider-status-'.$this->server->uuid))->expireAfter(130)->dontRelease()];
+ }
+
+ public function handle(): void
+ {
+ try {
+ if (! $this->server->cloudProviderToken) {
+ return;
+ }
+
+ match ($this->server->cloudProviderToken->provider) {
+ 'hetzner' => $this->server->hetzner_server_id
+ ? $this->server->refreshHetznerState()
+ : null,
+ 'vultr' => $this->server->vultr_instance_id
+ ? $this->server->refreshVultrState()
+ : null,
+ 'digitalocean' => $this->server->digitalocean_droplet_id
+ ? $this->server->refreshDigitalOceanState()
+ : null,
+ default => null,
+ };
+ } catch (\Throwable $e) {
+ Log::debug('Cloud provider status check failed', [
+ 'server_id' => $this->server->id,
+ 'error' => $e->getMessage(),
+ ]);
+ }
+ }
+}
diff --git a/app/Jobs/ServerConnectionCheckJob.php b/app/Jobs/ServerConnectionCheckJob.php
index 60adf8c180..f86686df7c 100644
--- a/app/Jobs/ServerConnectionCheckJob.php
+++ b/app/Jobs/ServerConnectionCheckJob.php
@@ -6,7 +6,6 @@ use App\Events\ServerReachabilityChanged;
use App\Helpers\SshMultiplexingHelper;
use App\Models\Server;
use App\Services\ConfigurationRepository;
-use App\Services\HetznerService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -42,8 +41,12 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
$configRepository->disableSshMux();
}
- public function handle()
+ public function handle(): void
{
+ if ($this->server->hasPlaceholderIp()) {
+ return;
+ }
+
$wasReachable = (bool) $this->server->settings->is_reachable;
$wasNotified = (bool) $this->server->unreachable_notification_sent;
@@ -62,19 +65,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
return;
}
- // Check Hetzner server status if applicable
- if ($this->server->hetzner_server_id && $this->server->cloudProviderToken) {
- $this->checkHetznerStatus();
- }
-
- if ($this->server->vultr_instance_id && $this->server->cloudProviderToken) {
- $this->checkVultrStatus();
- }
-
- if ($this->server->digitalocean_droplet_id && $this->server->cloudProviderToken) {
- $this->checkDigitalOceanStatus();
- }
-
// Temporarily disable mux if requested
if ($this->disableMux) {
$this->disableSshMux();
@@ -136,17 +126,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
public function failed(?\Throwable $exception): void
{
if ($exception instanceof TimeoutExceededException) {
- $wasReachable = (bool) $this->server->settings->is_reachable;
- $wasNotified = (bool) $this->server->unreachable_notification_sent;
-
- $this->server->settings->update([
- 'is_reachable' => false,
- 'is_usable' => false,
- ]);
- $this->server->increment('unreachable_count');
-
- $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
-
// Delete the queue job so it doesn't appear in Horizon's failed list.
$this->job?->delete();
}
@@ -171,63 +150,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
}
}
- private function checkHetznerStatus(): void
- {
- $status = null;
-
- try {
- $hetznerService = new HetznerService($this->server->cloudProviderToken->token);
- $serverData = $hetznerService->getServer($this->server->hetzner_server_id);
- $status = $serverData['status'] ?? null;
-
- } catch (\Throwable) {
- // Silently ignore — server may have been deleted from Hetzner.
- }
- if ($this->server->hetzner_server_status !== $status) {
- $this->server->update(['hetzner_server_status' => $status]);
- $this->server->hetzner_server_status = $status;
- if ($status === 'off') {
- throw new \Exception('Server is powered off');
- }
- }
-
- }
-
- private function checkVultrStatus(): void
- {
- try {
- $status = $this->server->refreshVultrState();
- } catch (\Throwable) {
- // Silently ignore transient Vultr API errors.
-
- return;
- }
-
- if (in_array($status, ['stopped', 'suspended', 'deleted'], true)) {
- throw new \Exception('Vultr instance is not running');
- }
- }
-
- private function checkDigitalOceanStatus(): void
- {
- try {
- $status = $this->server->refreshDigitalOceanState();
- } catch (\Throwable $e) {
- Log::debug('ServerConnectionCheck: DigitalOcean status check failed', [
- 'server_id' => $this->server->id,
- 'error' => $e->getMessage(),
- ]);
-
- return;
- }
-
- $this->server->digitalocean_droplet_status = $status;
-
- if (in_array($status, ['off', 'archive', 'deleted'], true)) {
- throw new \Exception('DigitalOcean droplet is not running');
- }
- }
-
private function checkConnection(): bool
{
try {
diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php
index 9532282cc5..67c222c24d 100644
--- a/app/Jobs/ServerManagerJob.php
+++ b/app/Jobs/ServerManagerJob.php
@@ -55,6 +55,13 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
// Get all servers to process
$servers = $this->getServers();
+ // Provider state checks run independently so slow APIs cannot block SSH checks.
+ $this->dispatchCloudProviderStatusChecks($servers);
+
+ $servers = $servers
+ ->reject(fn (Server $server) => $server->hasPlaceholderIp())
+ ->values();
+
// Dispatch ServerConnectionCheck for all servers efficiently
$this->dispatchConnectionChecks($servers);
@@ -64,24 +71,45 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
private function getServers(): Collection
{
- $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
+ $allServers = Server::with(['settings', 'cloudProviderToken']);
if (isCloud()) {
$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
- $own = Team::find(0)->servers()->with('settings')->get();
+ $own = Team::find(0)->servers()->with(['settings', 'cloudProviderToken'])->get();
- return $servers->merge($own);
+ return $servers->merge($own)->unique('id')->values();
} else {
return $allServers->get();
}
}
+ private function dispatchCloudProviderStatusChecks(Collection $servers): void
+ {
+ if (! shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime)) {
+ return;
+ }
+
+ $servers->each(function (Server $server) {
+ $hasCloudResource = $server->hetzner_server_id
+ || $server->vultr_instance_id
+ || $server->digitalocean_droplet_id;
+
+ if ($hasCloudResource && $server->cloudProviderToken) {
+ ServerCloudProviderStatusCheckJob::dispatch($server);
+ }
+ });
+ }
+
private function dispatchConnectionChecks(Collection $servers): void
{
if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
$servers->each(function (Server $server) {
try {
+ if ($server->hasPlaceholderIp()) {
+ return;
+ }
+
// Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
return;
diff --git a/app/Livewire/Server/New/ByDigitalOcean.php b/app/Livewire/Server/New/ByDigitalOcean.php
index a59f23efb0..cf27f0f081 100644
--- a/app/Livewire/Server/New/ByDigitalOcean.php
+++ b/app/Livewire/Server/New/ByDigitalOcean.php
@@ -14,6 +14,7 @@ use App\Services\DigitalOceanService;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Http\Client\RequestException;
use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
use Livewire\Attributes\Locked;
use Livewire\Component;
@@ -449,6 +450,10 @@ class ByDigitalOcean extends Component
{
$this->validate();
+ $digitalOceanService = null;
+ $dropletId = null;
+ $server = null;
+
try {
$this->authorize('create', Server::class);
@@ -468,24 +473,29 @@ class ByDigitalOcean extends Component
$digitalOceanService = new DigitalOceanService($this->getDigitalOceanToken());
$droplet = $this->createDigitalOceanDroplet($digitalOceanService);
+ $dropletId = (int) $droplet['id'];
// Persist the server immediately so the droplet is always tracked
// in Coolify, even if waiting for the public IP fails below.
- $server = Server::create([
- 'name' => strtolower(trim($this->server_name)),
- 'ip' => Server::PLACEHOLDER_IP,
- 'user' => 'root',
- 'port' => 22,
- 'team_id' => currentTeam()->id,
- 'private_key_id' => $this->private_key_id,
- 'cloud_provider_token_id' => $this->selected_token_id,
- 'digitalocean_droplet_id' => $droplet['id'],
- 'digitalocean_droplet_status' => $droplet['status'] ?? null,
- ]);
+ $server = DB::transaction(function () use ($dropletId, $droplet): Server {
+ $server = Server::create([
+ 'name' => strtolower(trim($this->server_name)),
+ 'ip' => Server::PLACEHOLDER_IP,
+ 'user' => 'root',
+ 'port' => 22,
+ 'team_id' => currentTeam()->id,
+ 'private_key_id' => $this->private_key_id,
+ 'cloud_provider_token_id' => $this->selected_token_id,
+ 'digitalocean_droplet_id' => $dropletId,
+ 'digitalocean_droplet_status' => $droplet['status'] ?? null,
+ ]);
- $server->proxy->set('status', 'exited');
- $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
- $server->save();
+ $server->proxy->set('status', 'exited');
+ $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
+ $server->save();
+
+ return $server;
+ });
try {
$droplet = $digitalOceanService->waitForPublicIp($droplet, true, $this->enable_ipv6);
@@ -510,10 +520,25 @@ class ByDigitalOcean extends Component
return redirectRoute($this, 'server.show', [$server->uuid]);
} catch (\Throwable $e) {
+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
+
return handleError($e, $this);
}
}
+ private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void
+ {
+ if (! $digitalOceanService || ! $dropletId || $server) {
+ return;
+ }
+
+ try {
+ $digitalOceanService->deleteDroplet($dropletId);
+ } catch (\Throwable $e) {
+ report($e);
+ }
+ }
+
public function render()
{
return view('livewire.server.new.by-digital-ocean');
diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.php
index 83f4f83d93..6cd0043624 100644
--- a/app/Livewire/Server/Show.php
+++ b/app/Livewire/Server/Show.php
@@ -611,6 +611,7 @@ class Show extends Component
public function startVultrInstance()
{
try {
+ $this->authorize('update', $this->server);
if (! $this->server->vultr_instance_id || ! $this->server->cloudProviderToken) {
$this->dispatch('error', 'This server is not associated with a Vultr instance or token.');
diff --git a/app/Models/Server.php b/app/Models/Server.php
index 928f80fed6..d55df0179c 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -18,6 +18,7 @@ use App\Notifications\Server\Reachable;
use App\Notifications\Server\Unreachable;
use App\Services\ConfigurationRepository;
use App\Services\DigitalOceanService;
+use App\Services\HetznerService;
use App\Services\VultrService;
use App\Support\ValidationPatterns;
use App\Traits\ClearsGlobalSearchCache;
@@ -119,6 +120,8 @@ class Server extends BaseModel
*/
public const PLACEHOLDER_IP = '1.2.3.4';
+ public const PLACEHOLDER_IPS = [self::PLACEHOLDER_IP, '0.0.0.0', '::'];
+
public static $batch_counter = 0;
/**
@@ -317,9 +320,12 @@ class Server extends BaseModel
public function hasPlaceholderIp(): bool
{
// Cast: the saving hook stores the ip as a Stringable in memory.
- $ip = (string) $this->ip;
+ return self::isPlaceholderIp((string) $this->ip);
+ }
- return blank($ip) || in_array($ip, [self::PLACEHOLDER_IP, '0.0.0.0', '::'], true);
+ public static function isPlaceholderIp(?string $ip): bool
+ {
+ return blank($ip) || in_array($ip, self::PLACEHOLDER_IPS, true);
}
/**
@@ -328,13 +334,66 @@ class Server extends BaseModel
*/
public function backfillPlaceholderIp(?string $ip): bool
{
- if ($ip && $this->hasPlaceholderIp()) {
- $this->update(['ip' => $ip]);
+ if (self::isPlaceholderIp($ip)) {
+ return false;
+ }
- return true;
+ $updated = static::query()
+ ->whereKey($this->getKey())
+ ->where(function (Builder $query): void {
+ $query->whereNull('ip')
+ ->orWhere('ip', '')
+ ->orWhereIn('ip', self::PLACEHOLDER_IPS);
+ })
+ ->update(['ip' => $ip]);
+
+ if ($updated === 0) {
+ return false;
}
- return false;
+ $this->forceFill(['ip' => $ip]);
+ $this->syncOriginalAttribute('ip');
+ static::flushIdentityMap();
+
+ return true;
+ }
+
+ /**
+ * Persist provider status without saving a stale in-memory IP value.
+ *
+ * @param array<string, mixed> $updates
+ */
+ private function persistProviderState(array $updates): void
+ {
+ if (empty($updates)) {
+ return;
+ }
+
+ static::query()->whereKey($this->getKey())->update($updates);
+ $this->forceFill($updates);
+ $this->syncOriginalAttributes(array_keys($updates));
+ static::flushIdentityMap();
+ }
+
+ public function refreshHetznerState(): ?string
+ {
+ if (! $this->hetzner_server_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'hetzner') {
+ return $this->hetzner_server_status;
+ }
+
+ $hetznerService = new HetznerService($this->cloudProviderToken->token);
+ $server = $hetznerService->getServer($this->hetzner_server_id);
+ $status = $server['status'] ?? null;
+ $assignedIp = data_get($server, 'public_net.ipv4.ip') ?? data_get($server, 'public_net.ipv6.ip');
+
+ $updates = [];
+ if ($this->hetzner_server_status !== $status) {
+ $updates['hetzner_server_status'] = $status;
+ }
+ $this->persistProviderState($updates);
+ $this->backfillPlaceholderIp($assignedIp);
+
+ return $status;
}
public function refreshVultrState(): ?string
@@ -352,8 +411,7 @@ class Server extends BaseModel
}
if ($this->vultr_instance_status !== 'deleted') {
- $this->update(['vultr_instance_status' => 'deleted']);
- $this->forceFill(['vultr_instance_status' => 'deleted']);
+ $this->persistProviderState(['vultr_instance_status' => 'deleted']);
}
return 'deleted';
@@ -368,15 +426,8 @@ class Server extends BaseModel
if ($this->vultr_instance_status !== $status) {
$updates['vultr_instance_status'] = $status;
}
-
- if ($this->hasPlaceholderIp() && $publicIp) {
- $updates['ip'] = $publicIp;
- }
-
- if (! empty($updates)) {
- $this->update($updates);
- $this->forceFill($updates);
- }
+ $this->persistProviderState($updates);
+ $this->backfillPlaceholderIp($publicIp);
return $status;
}
@@ -393,7 +444,7 @@ class Server extends BaseModel
$droplet = $digitalOceanService->getDroplet((int) $this->digitalocean_droplet_id);
} catch (RequestException $e) {
if ($e->response?->status() === 404) {
- $this->update(['digitalocean_droplet_status' => 'deleted']);
+ $this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
return 'deleted';
}
@@ -401,7 +452,7 @@ class Server extends BaseModel
throw $e;
} catch (\Throwable $e) {
if ((int) $e->getCode() === 404) {
- $this->update(['digitalocean_droplet_status' => 'deleted']);
+ $this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
return 'deleted';
}
@@ -416,12 +467,8 @@ class Server extends BaseModel
$status = $droplet['status'] ?? null;
$ip = $digitalOceanService->getPublicIpAddress($droplet);
- $updates = ['digitalocean_droplet_status' => $status];
- if ($ip && $this->hasPlaceholderIp()) {
- $updates['ip'] = $ip;
- }
-
- $this->update($updates);
+ $this->persistProviderState(['digitalocean_droplet_status' => $status]);
+ $this->backfillPlaceholderIp($ip);
return $status;
}
diff --git a/tests/Feature/DigitalOceanApiTest.php b/tests/Feature/DigitalOceanApiTest.php
index 04c1e16bea..8268999576 100644
--- a/tests/Feature/DigitalOceanApiTest.php
+++ b/tests/Feature/DigitalOceanApiTest.php
@@ -3,6 +3,7 @@
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
+use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -133,6 +134,109 @@ describe('POST /api/v1/servers/digitalocean', function () {
&& $request['user_data'] === '#cloud-config');
});
+ test('tracks the droplet with a placeholder IP when waiting for its IP fails', function () {
+ Http::fake([
+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([
+ 'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
+ ], 201),
+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([
+ 'ssh_keys' => [],
+ 'links' => ['pages' => []],
+ ], 200),
+ 'https://api.digitalocean.com/v2/droplets' => Http::response([
+ 'droplet' => [
+ 'id' => 987,
+ 'name' => 'waiting-for-ip',
+ 'status' => 'new',
+ 'networks' => ['v4' => [], 'v6' => []],
+ ],
+ ], 202),
+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+ 'message' => 'temporary provider failure',
+ ], 500),
+ ]);
+
+ $response = $this->withHeaders([
+ 'Authorization' => 'Bearer '.$this->bearerToken,
+ 'Content-Type' => 'application/json',
+ ])->postJson('/api/v1/servers/digitalocean', [
+ 'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
+ 'region' => 'nyc1',
+ 'size' => 's-1vcpu-1gb',
+ 'image' => 'ubuntu-24-04-x64',
+ 'name' => 'waiting-for-ip',
+ 'private_key_uuid' => $this->privateKey->uuid,
+ ]);
+
+ $response->assertCreated();
+ $response->assertJsonFragment([
+ 'digitalocean_droplet_id' => 987,
+ 'ip' => Server::PLACEHOLDER_IP,
+ ]);
+
+ $this->assertDatabaseHas('servers', [
+ 'name' => 'waiting-for-ip',
+ 'ip' => Server::PLACEHOLDER_IP,
+ 'team_id' => $this->team->id,
+ 'digitalocean_droplet_id' => 987,
+ 'digitalocean_droplet_status' => 'new',
+ ]);
+ });
+
+ test('deletes the droplet when local server persistence fails', function () {
+ Http::fake([
+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([
+ 'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
+ ], 201),
+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([
+ 'ssh_keys' => [],
+ 'links' => ['pages' => []],
+ ], 200),
+ 'https://api.digitalocean.com/v2/droplets' => Http::response([
+ 'droplet' => [
+ 'id' => 987,
+ 'name' => 'persistence-fails',
+ 'status' => 'active',
+ 'networks' => [
+ 'v4' => [
+ ['ip_address' => '203.0.113.10', 'type' => 'public'],
+ ],
+ ],
+ ],
+ ], 202),
+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response(null, 204),
+ ]);
+
+ $eventDispatcher = Server::getEventDispatcher();
+ Server::setEventDispatcher(clone $eventDispatcher);
+ Server::created(function (): void {
+ throw new RuntimeException('local persistence failed');
+ });
+
+ try {
+ $response = $this->withHeaders([
+ 'Authorization' => 'Bearer '.$this->bearerToken,
+ 'Content-Type' => 'application/json',
+ ])->postJson('/api/v1/servers/digitalocean', [
+ 'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
+ 'region' => 'nyc1',
+ 'size' => 's-1vcpu-1gb',
+ 'image' => 'ubuntu-24-04-x64',
+ 'name' => 'persistence-fails',
+ 'private_key_uuid' => $this->privateKey->uuid,
+ ]);
+ } finally {
+ Server::setEventDispatcher($eventDispatcher);
+ }
+
+ $response->assertServerError();
+ $this->assertDatabaseMissing('servers', [
+ 'digitalocean_droplet_id' => 987,
+ ]);
+ Http::assertSent(fn ($request) => $request->method() === 'DELETE'
+ && $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
+ });
+
test('validates required fields', function () {
$response = $this->withHeaders([
'Authorization' => 'Bearer '.$this->bearerToken,
diff --git a/tests/Feature/DigitalOceanServerCreationTest.php b/tests/Feature/DigitalOceanServerCreationTest.php
index b42148dfc4..3d81a5dc97 100644
--- a/tests/Feature/DigitalOceanServerCreationTest.php
+++ b/tests/Feature/DigitalOceanServerCreationTest.php
@@ -113,6 +113,54 @@ it('updates the placeholder IP once the droplet reports one', function () {
]);
});
+it('deletes the droplet when local server persistence fails', function () {
+ Http::fake([
+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([
+ 'ssh_key' => ['id' => 123],
+ ], 201),
+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([
+ 'ssh_keys' => [],
+ ], 200),
+ 'https://api.digitalocean.com/v2/droplets' => Http::response([
+ 'droplet' => [
+ 'id' => 555,
+ 'status' => 'active',
+ 'networks' => [
+ 'v4' => [
+ ['type' => 'public', 'ip_address' => '203.0.113.40'],
+ ],
+ ],
+ ],
+ ], 202),
+ 'https://api.digitalocean.com/v2/droplets/555' => Http::response(null, 204),
+ ]);
+
+ $eventDispatcher = Server::getEventDispatcher();
+ Server::setEventDispatcher(clone $eventDispatcher);
+ Server::created(function (): void {
+ throw new RuntimeException('local persistence failed');
+ });
+
+ try {
+ Livewire::test(ByDigitalOcean::class, ['selectedTokenUuid' => $this->digitalOceanToken->uuid])
+ ->set('server_name', 'persistence-fails')
+ ->set('selected_region', 'nyc1')
+ ->set('selected_size', 's-1vcpu-1gb')
+ ->set('selected_image', 'ubuntu-24-04-x64')
+ ->set('private_key_id', $this->privateKey->id)
+ ->call('submit')
+ ->assertDispatched('error', 'local persistence failed');
+ } finally {
+ Server::setEventDispatcher($eventDispatcher);
+ }
+
+ $this->assertDatabaseMissing('servers', [
+ 'digitalocean_droplet_id' => 555,
+ ]);
+ Http::assertSent(fn ($request) => $request->method() === 'DELETE'
+ && $request->url() === 'https://api.digitalocean.com/v2/droplets/555');
+});
+
it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {
Livewire::test(ByDigitalOcean::class)
->set('current_step', 2)
diff --git a/tests/Feature/ServerCloudProviderStatusCheckJobTest.php b/tests/Feature/ServerCloudProviderStatusCheckJobTest.php
new file mode 100644
index 0000000000..10e1a8c9d8
--- /dev/null
+++ b/tests/Feature/ServerCloudProviderStatusCheckJobTest.php
@@ -0,0 +1,149 @@
+<?php
+
+use App\Jobs\ServerCloudProviderStatusCheckJob;
+use App\Jobs\ServerConnectionCheckJob;
+use App\Jobs\ServerManagerJob;
+use App\Models\CloudProviderToken;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+function createCloudServerForStatusCheckJobTest(string $provider, array $attributes = []): Server
+{
+ $team = Team::factory()->create();
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $token = CloudProviderToken::create([
+ 'team_id' => $team->id,
+ 'provider' => $provider,
+ 'token' => "test-{$provider}-token",
+ 'name' => ucfirst($provider),
+ ]);
+
+ return Server::factory()->create(array_merge([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ 'cloud_provider_token_id' => $token->id,
+ 'ip' => Server::PLACEHOLDER_IP,
+ ], $attributes));
+}
+
+beforeEach(function () {
+ InstanceSettings::forceCreate([
+ 'id' => 0,
+ 'instance_timezone' => 'UTC',
+ ]);
+ Carbon::setTestNow('2026-07-11 12:00:00');
+});
+
+afterEach(function () {
+ Carbon::setTestNow();
+});
+
+it('syncs provider state for a placeholder server without scheduling SSH', function () {
+ $server = createCloudServerForStatusCheckJobTest('digitalocean', [
+ 'digitalocean_droplet_id' => 987,
+ 'digitalocean_droplet_status' => 'new',
+ ]);
+
+ Queue::fake();
+
+ (new ServerManagerJob)->handle();
+
+ Queue::assertPushed(ServerCloudProviderStatusCheckJob::class, fn (ServerCloudProviderStatusCheckJob $job) => $job->server->is($server));
+ Queue::assertNotPushed(ServerConnectionCheckJob::class);
+});
+
+it('backfills a DigitalOcean placeholder without changing reachability', function () {
+ $server = createCloudServerForStatusCheckJobTest('digitalocean', [
+ 'digitalocean_droplet_id' => 987,
+ 'digitalocean_droplet_status' => 'new',
+ ]);
+ $server->settings->update([
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ ]);
+
+ Http::fake([
+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+ 'droplet' => [
+ 'id' => 987,
+ 'status' => 'active',
+ 'networks' => [
+ 'v4' => [
+ ['type' => 'public', 'ip_address' => '203.0.113.10'],
+ ],
+ ],
+ ],
+ ]),
+ ]);
+
+ (new ServerCloudProviderStatusCheckJob($server))->handle();
+
+ $server->refresh();
+ expect($server->ip)->toBe('203.0.113.10')
+ ->and($server->digitalocean_droplet_status)->toBe('active')
+ ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
+ ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
+});
+
+it('backfills a Hetzner placeholder without changing reachability', function () {
+ $server = createCloudServerForStatusCheckJobTest('hetzner', [
+ 'hetzner_server_id' => 123,
+ 'hetzner_server_status' => 'starting',
+ ]);
+ $server->settings->update([
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ ]);
+
+ Http::fake([
+ 'https://api.hetzner.cloud/v1/servers/123' => Http::response([
+ 'server' => [
+ 'id' => 123,
+ 'status' => 'running',
+ 'public_net' => [
+ 'ipv4' => ['ip' => '198.51.100.20'],
+ ],
+ ],
+ ]),
+ ]);
+
+ (new ServerCloudProviderStatusCheckJob($server))->handle();
+
+ $server->refresh();
+ expect($server->ip)->toBe('198.51.100.20')
+ ->and($server->hetzner_server_status)->toBe('running')
+ ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
+ ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
+});
+
+it('only calls the API matching the cloud provider token', function () {
+ $server = createCloudServerForStatusCheckJobTest('digitalocean', [
+ 'vultr_instance_id' => 'unrelated-vultr-instance',
+ 'digitalocean_droplet_id' => 987,
+ 'digitalocean_droplet_status' => 'new',
+ ]);
+
+ Http::fake([
+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+ 'droplet' => [
+ 'id' => 987,
+ 'status' => 'active',
+ 'networks' => ['v4' => []],
+ ],
+ ]),
+ 'https://api.vultr.com/*' => Http::response(status: 500),
+ ]);
+
+ (new ServerCloudProviderStatusCheckJob($server))->handle();
+
+ Http::assertSentCount(1);
+ Http::assertSent(fn ($request) => $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
+});
diff --git a/tests/Feature/ServerConnectionCheckIsolationTest.php b/tests/Feature/ServerConnectionCheckIsolationTest.php
new file mode 100644
index 0000000000..d098c39b55
--- /dev/null
+++ b/tests/Feature/ServerConnectionCheckIsolationTest.php
@@ -0,0 +1,80 @@
+<?php
+
+use App\Jobs\ServerConnectionCheckJob;
+use App\Models\CloudProviderToken;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+function createServerForConnectionIsolationTest(array $attributes = []): Server
+{
+ $team = Team::factory()->create();
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+
+ return Server::factory()->create(array_merge([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ ], $attributes));
+}
+
+beforeEach(function () {
+ Storage::fake('ssh-keys');
+});
+
+it('never attempts SSH to a placeholder address', function (string $placeholderIp) {
+ $server = createServerForConnectionIsolationTest(['ip' => $placeholderIp]);
+ $server->settings->update([
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ ]);
+
+ Process::fake();
+
+ (new ServerConnectionCheckJob($server, disableMux: false))->handle();
+
+ Process::assertNothingRan();
+ expect($server->fresh()->unreachable_count)->toBe(0)
+ ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
+ ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
+})->with([
+ 'reserved provisioning address' => Server::PLACEHOLDER_IP,
+ 'unspecified IPv4 address' => '0.0.0.0',
+ 'unspecified IPv6 address' => '::',
+]);
+
+it('does not call cloud provider APIs during an SSH connection check', function () {
+ $server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
+ $token = CloudProviderToken::create([
+ 'team_id' => $server->team_id,
+ 'provider' => 'vultr',
+ 'token' => 'test-vultr-token',
+ 'name' => 'Vultr',
+ ]);
+ $server->update([
+ 'cloud_provider_token_id' => $token->id,
+ 'vultr_instance_id' => 'instance-1',
+ 'vultr_instance_status' => 'active',
+ ]);
+
+ Http::fake([
+ 'https://api.vultr.com/*' => Http::response([
+ 'instance' => ['id' => 'instance-1', 'status' => 'active'],
+ ]),
+ ]);
+ Process::fake([
+ '*' => Process::result(
+ output: '{"Server":{"Version":"27.0.0"}}',
+ exitCode: 0,
+ ),
+ ]);
+
+ (new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
+
+ Http::assertNothingSent();
+});
diff --git a/tests/Feature/VultrApiTest.php b/tests/Feature/VultrApiTest.php
index 8a96ec6857..9eb135baf5 100644
--- a/tests/Feature/VultrApiTest.php
+++ b/tests/Feature/VultrApiTest.php
@@ -1,11 +1,15 @@
<?php
+use App\Http\Controllers\Api\VultrController;
use App\Models\CloudProviderToken;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
+use App\Models\Server;
use App\Models\Team;
use App\Models\User;
+use Illuminate\Auth\Access\AuthorizationException;
use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Request;
use Illuminate\Support\Facades\Http;
uses(RefreshDatabase::class);
@@ -165,6 +169,27 @@ describe('GET /api/v1/vultr/ssh-keys', function () {
});
});
+test('member read token cannot use a stored Vultr token for metadata endpoints', function (string $endpoint) {
+ $member = User::factory()->create();
+ $this->team->members()->attach($member->id, ['role' => 'member']);
+ $memberToken = $member->createToken('member-read', ['read'])->plainTextToken;
+
+ Http::fake();
+
+ $response = $this->withHeaders([
+ 'Authorization' => 'Bearer '.$memberToken,
+ 'Content-Type' => 'application/json',
+ ])->getJson($endpoint.'?cloud_provider_token_uuid='.$this->vultrToken->uuid);
+
+ $response->assertForbidden();
+ Http::assertNothingSent();
+})->with([
+ '/api/v1/vultr/regions',
+ '/api/v1/vultr/plans',
+ '/api/v1/vultr/os',
+ '/api/v1/vultr/ssh-keys',
+]);
+
describe('POST /api/v1/servers/vultr', function () {
test('creates a Vultr server', function () {
Http::fake([
@@ -223,6 +248,11 @@ describe('POST /api/v1/servers/vultr', function () {
});
test('waits for Vultr to assign a real public IP when creation returns placeholder IP', function () {
+ $createdIp = null;
+ Server::created(function (Server $server) use (&$createdIp): void {
+ $createdIp = (string) $server->ip;
+ });
+
Http::fake([
'https://api.vultr.com/v2/ssh-keys' => Http::response([
'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
@@ -262,6 +292,8 @@ describe('POST /api/v1/servers/vultr', function () {
$response->assertStatus(201);
$response->assertJsonFragment(['ip' => '1.2.3.4']);
+ expect($createdIp)->toBe(Server::PLACEHOLDER_IP);
+
$this->assertDatabaseHas('servers', [
'name' => 'test-server',
'ip' => '1.2.3.4',
@@ -269,6 +301,34 @@ describe('POST /api/v1/servers/vultr', function () {
]);
});
+ test('server creation authorizes access to the stored Vultr token', function () {
+ $member = User::factory()->create();
+ $this->team->members()->attach($member->id, ['role' => 'member']);
+ $this->actingAs($member);
+ $member->withAccessToken($member->createToken('member-all', ['*'])->accessToken);
+
+ $request = Request::create(
+ uri: '/api/v1/servers/vultr',
+ method: 'POST',
+ server: ['CONTENT_TYPE' => 'application/json'],
+ content: json_encode([
+ 'cloud_provider_token_uuid' => $this->vultrToken->uuid,
+ 'region' => 'ewr',
+ 'plan' => 'vc2-1c-1gb',
+ 'os_id' => 2284,
+ 'name' => 'test-server',
+ 'private_key_uuid' => $this->privateKey->uuid,
+ ], JSON_THROW_ON_ERROR),
+ );
+ $request->setUserResolver(fn () => $member);
+
+ Http::fake();
+
+ expect(fn () => app(VultrController::class)->createServer($request))
+ ->toThrow(AuthorizationException::class);
+ Http::assertNothingSent();
+ });
+
test('reuses existing matching Vultr SSH key', function () {
Http::fake([
'https://api.vultr.com/v2/ssh-keys*' => Http::response([
diff --git a/tests/Feature/VultrServerLifecycleTest.php b/tests/Feature/VultrServerLifecycleTest.php
index c89ca47b45..cb3b3108c0 100644
--- a/tests/Feature/VultrServerLifecycleTest.php
+++ b/tests/Feature/VultrServerLifecycleTest.php
@@ -123,6 +123,8 @@ it('updates placeholder server IP when Vultr status refresh returns assigned pub
});
it('does not overwrite an established server IP during Vultr status refresh', function () {
+ $this->server->update(['ip' => '198.51.100.20']);
+
Http::fake([
'https://api.vultr.com/v2/instances/instance-1' => Http::response([
'instance' => [
@@ -135,12 +137,12 @@ it('does not overwrite an established server IP during Vultr status refresh', fu
Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
->call('checkVultrInstanceStatus', true)
- ->assertSet('ip', '1.2.3.4')
+ ->assertSet('ip', '198.51.100.20')
->assertSet('vultrInstanceStatus', 'active');
$this->assertDatabaseHas('servers', [
'id' => $this->server->id,
- 'ip' => '1.2.3.4',
+ 'ip' => '198.51.100.20',
'vultr_instance_status' => 'active',
]);
});
@@ -270,6 +272,26 @@ it('starts a Vultr instance', function () {
Http::assertSent(fn ($request) => $request->url() === 'https://api.vultr.com/v2/instances/instance-1/start');
});
+it('prevents members from starting a Vultr instance', function () {
+ $this->server->update(['vultr_instance_status' => 'stopped']);
+
+ $member = User::factory()->create();
+ $this->team->members()->attach($member->id, ['role' => 'member']);
+ $this->actingAs($member);
+ session(['currentTeam' => $this->team]);
+
+ Http::fake([
+ 'https://api.vultr.com/v2/instances/instance-1/start' => Http::response([], 204),
+ ]);
+
+ Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
+ ->call('startVultrInstance')
+ ->assertDispatched('error');
+
+ expect($this->server->fresh()->vultr_instance_status)->toBe('stopped');
+ Http::assertNothingSent();
+});
+
it('links a server to Vultr by matching IP', function () {
$unlinkedServer = Server::factory()->create([
'team_id' => $this->team->id,
diff --git a/tests/Unit/DigitalOceanDeleteServerTest.php b/tests/Unit/DigitalOceanDeleteServerTest.php
index 5b0989607e..60cd5ba502 100644
--- a/tests/Unit/DigitalOceanDeleteServerTest.php
+++ b/tests/Unit/DigitalOceanDeleteServerTest.php
@@ -6,6 +6,7 @@ use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\RequestException;
use Illuminate\Support\Facades\Http;
use Tests\TestCase;
@@ -44,3 +45,59 @@ it('deletes the DigitalOcean droplet when requested', function () {
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
});
+
+it('retains the server and surfaces a DigitalOcean deletion failure', function () {
+ Http::fake([
+ 'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+ 'message' => 'deletion failed',
+ ], 500),
+ ]);
+
+ $team = Team::factory()->create();
+ $token = CloudProviderToken::factory()->create([
+ 'team_id' => $team->id,
+ 'provider' => 'digitalocean',
+ 'token' => 'test-digitalocean-token',
+ ]);
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ 'cloud_provider_token_id' => $token->id,
+ 'digitalocean_droplet_id' => 987,
+ ]);
+ $server->delete();
+
+ expect(fn () => DeleteServer::run(
+ serverId: $server->id,
+ deleteFromDigitalOcean: true,
+ digitalOceanDropletId: 987,
+ cloudProviderTokenId: $token->id,
+ teamId: $team->id,
+ ))->toThrow(RequestException::class, 'status code 500');
+
+ expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
+
+it('retains the server when no DigitalOcean token can delete the droplet', function () {
+ Http::preventStrayRequests();
+
+ $team = Team::factory()->create();
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ 'cloud_provider_token_id' => null,
+ 'digitalocean_droplet_id' => 987,
+ ]);
+ $server->delete();
+
+ expect(fn () => DeleteServer::run(
+ serverId: $server->id,
+ deleteFromDigitalOcean: true,
+ digitalOceanDropletId: 987,
+ teamId: $team->id,
+ ))->toThrow(RuntimeException::class, 'No DigitalOcean token found');
+
+ expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
diff --git a/tests/Unit/DigitalOceanServerStateTest.php b/tests/Unit/DigitalOceanServerStateTest.php
index 93b3379bb9..725a38c5bf 100644
--- a/tests/Unit/DigitalOceanServerStateTest.php
+++ b/tests/Unit/DigitalOceanServerStateTest.php
@@ -66,8 +66,8 @@ it('backfills a placeholder IP from the DigitalOcean droplet state', function ()
expect($server->fresh()->ip)->toBe('203.0.113.10');
});
-it('does not overwrite a real IP from the DigitalOcean droplet state', function () {
- $server = createDigitalOceanServerForStateTest('active', ['ip' => '198.51.100.20']);
+it('does not overwrite an administrator configured address from the DigitalOcean droplet state', function (string $configuredAddress) {
+ $server = createDigitalOceanServerForStateTest('active', ['ip' => $configuredAddress]);
Http::fake([
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
@@ -84,7 +84,38 @@ it('does not overwrite a real IP from the DigitalOcean droplet state', function
]);
$server->refreshDigitalOceanState();
- expect($server->fresh()->ip)->toBe('198.51.100.20');
+ expect($server->fresh()->ip)->toBe($configuredAddress);
+})->with([
+ 'public IPv4' => '198.51.100.20',
+ 'private IPv4' => '10.10.0.12',
+ 'IPv6' => '2001:db8::10',
+ 'tunnel hostname' => 'server.internal.example.com',
+]);
+
+it('does not overwrite an address configured while DigitalOcean state is loading', function () {
+ $server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);
+
+ Http::fake(function () use ($server) {
+ Server::query()->findOrFail($server->id)->update([
+ 'ip' => 'server.internal.example.com',
+ ]);
+
+ return Http::response([
+ 'droplet' => [
+ 'id' => 987,
+ 'status' => 'active',
+ 'networks' => [
+ 'v4' => [
+ ['type' => 'public', 'ip_address' => '203.0.113.10'],
+ ],
+ ],
+ ],
+ ], 200);
+ });
+
+ expect($server->refreshDigitalOceanState())->toBe('active')
+ ->and($server->fresh()->ip)->toBe('server.internal.example.com')
+ ->and($server->digitalocean_droplet_status)->toBe('active');
});
it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {
diff --git a/tests/Unit/ServerBackoffTest.php b/tests/Unit/ServerBackoffTest.php
index 7c9910a5cb..3bbf41b3a2 100644
--- a/tests/Unit/ServerBackoffTest.php
+++ b/tests/Unit/ServerBackoffTest.php
@@ -127,7 +127,7 @@ describe('shouldSkipDueToBackoff', function () {
});
describe('ServerConnectionCheckJob unreachable_count', function () {
- it('marks Vultr servers unreachable when provider status is unavailable', function () {
+ it('marks servers unreachable when SSH is unavailable', function () {
Event::fake([ServerReachabilityChanged::class]);
$settings = Mockery::mock();
@@ -141,14 +141,12 @@ describe('ServerConnectionCheckJob unreachable_count', function () {
$server->shouldReceive('getAttribute')->andReturnUsing(fn (string $key) => match ($key) {
'settings' => $settings,
'unreachable_notification_sent' => false,
- 'vultr_instance_id' => 'instance-1',
- 'cloudProviderToken' => (object) ['token' => 'test-token'],
+ 'ip' => '203.0.113.10',
'id' => 1,
'name' => 'test-server',
'unreachable_count' => 1,
default => null,
});
- $server->shouldReceive('refreshVultrState')->once()->andReturn('stopped');
$server->shouldReceive('increment')->with('unreachable_count')->once();
$server->id = 1;
$server->name = 'test-server';
@@ -158,22 +156,20 @@ describe('ServerConnectionCheckJob unreachable_count', function () {
$job->handle();
});
- it('increments unreachable_count on timeout', function () {
+ it('does not change reachability on a job-level timeout', function () {
Event::fake([ServerReachabilityChanged::class]);
$settings = Mockery::mock();
$settings->is_reachable = true;
- $settings->shouldReceive('update')
- ->with(['is_reachable' => false, 'is_usable' => false])
- ->once();
+ $settings->shouldNotReceive('update');
$server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
$server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
$server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);
- $server->shouldReceive('increment')->with('unreachable_count')->once();
+ $server->shouldNotReceive('increment');
$server->id = 1;
$server->name = 'test-server';
- $server->unreachable_count = 1; // Will become 2 after increment in real code; mock keeps value as-is
+ $server->unreachable_count = 1;
$job = new ServerConnectionCheckJob($server);
$job->failed(new TimeoutExceededException);
diff --git a/tests/Unit/ServerPlaceholderIpTest.php b/tests/Unit/ServerPlaceholderIpTest.php
index b471ae7e13..94787b4d47 100644
--- a/tests/Unit/ServerPlaceholderIpTest.php
+++ b/tests/Unit/ServerPlaceholderIpTest.php
@@ -1,5 +1,7 @@
<?php
+use App\Jobs\CleanupOrphanedPreviewContainersJob;
+use App\Jobs\ScheduledJobManager;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
@@ -60,8 +62,64 @@ it('ignores a missing IP when backfilling', function () {
expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
});
+it('does not replace a placeholder with another placeholder', function (string $replacementIp) {
+ $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+ expect($server->backfillPlaceholderIp($replacementIp))->toBeFalse();
+ expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
+})->with([
+ 'unspecified IPv4 address' => '0.0.0.0',
+ 'unspecified IPv6 address' => '::',
+]);
+
+it('does not overwrite an address configured after the placeholder model was loaded', function () {
+ $staleServer = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+ $concurrentServer = Server::query()->findOrFail($staleServer->id);
+
+ $concurrentServer->update(['ip' => 'server.internal.example.com']);
+
+ expect($staleServer->backfillPlaceholderIp('203.0.113.10'))->toBeFalse()
+ ->and($staleServer->fresh()->ip)->toBe('server.internal.example.com');
+});
+
it('skips servers with a placeholder IP in scheduled jobs', function () {
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
expect($server->skipServer())->toBeTrue();
});
+
+it('excludes every placeholder address from scheduled Docker cleanup', function () {
+ $realServer = createServerForPlaceholderIpTest('203.0.113.10');
+
+ foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
+ Server::factory()->create([
+ 'team_id' => $realServer->team_id,
+ 'private_key_id' => $realServer->private_key_id,
+ 'ip' => $placeholderIp,
+ ]);
+ }
+
+ $method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');
+ $servers = $method->invoke(new ScheduledJobManager)->get();
+
+ expect($servers->modelKeys())->toBe([$realServer->id]);
+});
+
+it('excludes every placeholder address from orphaned preview cleanup', function () {
+ $realServer = createServerForPlaceholderIpTest('203.0.113.10');
+ $realServer->settings->update(['is_reachable' => true, 'is_usable' => true]);
+
+ foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
+ $server = Server::factory()->create([
+ 'team_id' => $realServer->team_id,
+ 'private_key_id' => $realServer->private_key_id,
+ 'ip' => $placeholderIp,
+ ]);
+ $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+ }
+
+ $method = new ReflectionMethod(CleanupOrphanedPreviewContainersJob::class, 'getServersToCheck');
+ $servers = $method->invoke(new CleanupOrphanedPreviewContainersJob);
+
+ expect($servers->modelKeys())->toBe([$realServer->id]);
+});
diff --git a/tests/Unit/VultrDeleteServerTest.php b/tests/Unit/VultrDeleteServerTest.php
index 05891eed7c..82173d40d4 100644
--- a/tests/Unit/VultrDeleteServerTest.php
+++ b/tests/Unit/VultrDeleteServerTest.php
@@ -7,6 +7,7 @@ use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\RequestException;
use Illuminate\Support\Facades\Http;
use Tests\TestCase;
@@ -116,3 +117,52 @@ it('does not use another team Vultr token when deleting an instance', function (
expect($request->header('Authorization'))->toBe(['Bearer test-vultr-token']);
});
+
+it('retains the server and surfaces a Vultr deletion failure', function () {
+ Http::fake([
+ 'https://api.vultr.com/v2/instances/instance-1' => Http::response([
+ 'error' => 'deletion failed',
+ ], 500),
+ ]);
+
+ $server = Server::factory()->create([
+ 'team_id' => $this->team->id,
+ 'private_key_id' => $this->privateKey->id,
+ 'cloud_provider_token_id' => $this->vultrToken->id,
+ 'vultr_instance_id' => 'instance-1',
+ ]);
+ $server->delete();
+
+ expect(fn () => DeleteServer::run(
+ serverId: $server->id,
+ cloudProviderTokenId: $this->vultrToken->id,
+ teamId: $this->team->id,
+ deleteFromVultr: true,
+ vultrInstanceId: 'instance-1'
+ ))->toThrow(RequestException::class, 'status code 500');
+
+ expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
+
+it('retains the server when no Vultr token can delete the instance', function () {
+ Http::preventStrayRequests();
+
+ $this->vultrToken->delete();
+
+ $server = Server::factory()->create([
+ 'team_id' => $this->team->id,
+ 'private_key_id' => $this->privateKey->id,
+ 'cloud_provider_token_id' => null,
+ 'vultr_instance_id' => 'instance-1',
+ ]);
+ $server->delete();
+
+ expect(fn () => DeleteServer::run(
+ serverId: $server->id,
+ teamId: $this->team->id,
+ deleteFromVultr: true,
+ vultrInstanceId: 'instance-1'
+ ))->toThrow(RuntimeException::class, 'No Vultr token found');
+
+ expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});