An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(cloud-providers): show why Hetzner, DigitalOcean, and Vultr API requests fail
Andras Bacsai16c6de823ba1
diff --git a/app/Exceptions/CloudProviderApiException.php b/app/Exceptions/CloudProviderApiException.phpnew file mode 100644index 0000000000..287d4aafd9--- /dev/null+++ b/app/Exceptions/CloudProviderApiException.php@@ -0,0 +1,73 @@+<?php++namespace App\Exceptions;++use Exception;+use Illuminate\Http\JsonResponse;++/**+ * An error response from a cloud provider API (Hetzner, DigitalOcean, Vultr).+ *+ * @see https://docs.hetzner.cloud/reference/cloud#errors+ */+class CloudProviderApiException extends Exception+{+ public readonly ?string $errorCode;++ /** @var array<int, string> */+ public readonly array $invalidFields;++ /**+ * The error code and field names come from the provider, so only values that look like identifiers are kept.+ *+ * @param array<int, mixed> $invalidFields+ */+ public function __construct(+ public readonly string $provider,+ string $providerMessage,+ public readonly int $status,+ mixed $errorCode = null,+ array $invalidFields = [],+ ) {+ parent::__construct("{$provider} API error: {$providerMessage}", $status);++ $errorCode = is_string($errorCode) ? strtolower($errorCode) : null;+ $this->errorCode = $errorCode !== null && preg_match('/^[a-z_]{1,64}$/', $errorCode) ? $errorCode : null;+ $this->invalidFields = array_values(array_filter(+ $invalidFields,+ fn ($name) => is_string($name) && preg_match('/^[a-z0-9_.\[\]]{1,64}$/i', $name)+ ));+ }++ /**+ * A message that is safe to return to API clients. It is built only from the+ * HTTP status, the error code, and field names, never from the provider's free-text message.+ */+ public function safeMessage(): string+ {+ return match (true) {+ $this->errorCode === 'resource_limit_exceeded' => "Your {$this->provider} account has reached a resource limit, for example the server limit. Remove unused resources or ask {$this->provider} for a higher limit.",+ $this->errorCode === 'resource_unavailable' => 'The selected server type, image, or location is not available now. Select a different location or server type.',+ $this->errorCode === 'uniqueness_error' => "A resource with the same name already exists in your {$this->provider} account, for example a server or an SSH key. Use a different name.",+ $this->errorCode === 'maintenance', $this->status === 423, $this->status >= 500 => "{$this->provider} cannot process the request now. Try again later.",+ $this->status === 401 => "The {$this->provider} API token is invalid or expired. Update the cloud provider token in Coolify.",+ $this->status === 403 => "The {$this->provider} API token does not have permission for this request. Use a token with read and write access.",+ $this->status === 404 => "{$this->provider} could not find a requested resource, for example the location, server type, image, or SSH key.",+ $this->status === 409 => "{$this->provider} rejected the request because of a conflict with an existing resource.",+ $this->status === 422 => "{$this->provider} rejected the request as invalid. Check the selected values and your account limits.".($this->invalidFields !== [] ? ' Invalid fields: '.implode(', ', $this->invalidFields).'.' : ''),+ default => "{$this->provider} returned HTTP {$this->status}".($this->errorCode !== null ? " (error code: {$this->errorCode})" : '').'.',+ };+ }++ /**+ * The API response for a failed provider request: 502, because the upstream API rejected it.+ */+ public function toJsonResponse(string $failureMessage): JsonResponse+ {+ return response()->json([+ 'message' => $failureMessage.' '.$this->safeMessage(),+ 'provider_status' => $this->status,+ 'provider_error_code' => $this->errorCode,+ ], 502);+ }+}diff --git a/app/Http/Controllers/Api/DigitalOceanController.php b/app/Http/Controllers/Api/DigitalOceanController.phpindex 427a805278..7c7dd6e00b 100644--- a/app/Http/Controllers/Api/DigitalOceanController.php+++ b/app/Http/Controllers/Api/DigitalOceanController.php@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;use App\Actions\Server\ValidateServer;use App\Enums\ProxyTypes;+use App\Exceptions\CloudProviderApiException;use App\Exceptions\RateLimitException;use App\Http\Controllers\Api\Concerns\RequiresDeployAbility;use App\Http\Controllers\Controller;@@ -210,6 +211,7 @@ class DigitalOceanController extends Controllernew OA\Response(response: 403, description: 'The token needs the deploy permission for a cloud-init script or extra SSH keys.'),new OA\Response(response: 422, description: 'Validation failed.'),new OA\Response(response: 429, description: 'DigitalOcean rate limit exceeded.'),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function createServer(Request $request): JsonResponse@@ -395,12 +397,13 @@ class DigitalOceanController extends Controller}return $response;- } catch (\Throwable $e) {+ } catch (CloudProviderApiException $e) {$this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);- logger()->error('Failed to create DigitalOcean server', [- 'error' => $e->getMessage(),- ]);+ return $e->toJsonResponse('Failed to create DigitalOcean server.');+ } catch (\Throwable $e) {+ $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);+ report($e);return response()->json(['message' => 'Failed to create DigitalOcean server.'], 500);}diff --git a/app/Http/Controllers/Api/HetznerController.php b/app/Http/Controllers/Api/HetznerController.phpindex ca0ec99116..509558a00f 100644--- a/app/Http/Controllers/Api/HetznerController.php+++ b/app/Http/Controllers/Api/HetznerController.php@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;use App\Actions\Server\ValidateServer;use App\Enums\ProxyTypes;+use App\Exceptions\CloudProviderApiException;use App\Exceptions\RateLimitException;use App\Http\Controllers\Api\Concerns\RequiresDeployAbility;use App\Http\Controllers\Controller;@@ -90,6 +91,7 @@ class HetznerController extends Controllerresponse: 404,ref: '#/components/responses/404',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function locations(Request $request)@@ -127,7 +129,11 @@ class HetznerController extends Controller$locations = $hetznerService->getLocations();return response()->json($locations);+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to fetch Hetzner locations.');} catch (\Throwable $e) {+ report($e);+return response()->json(['message' => 'Failed to fetch Hetzner locations.'], 500);}}@@ -212,6 +218,7 @@ class HetznerController extends Controllerresponse: 404,ref: '#/components/responses/404',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function serverTypes(Request $request)@@ -249,7 +256,11 @@ class HetznerController extends Controller$serverTypes = $hetznerService->getServerTypes();return response()->json($serverTypes);+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to fetch Hetzner server types.');} catch (\Throwable $e) {+ report($e);+return response()->json(['message' => 'Failed to fetch Hetzner server types.'], 500);}}@@ -312,6 +323,7 @@ class HetznerController extends Controllerresponse: 404,ref: '#/components/responses/404',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function images(Request $request)@@ -362,7 +374,11 @@ class HetznerController extends Controller});return response()->json(array_values($filtered));+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to fetch Hetzner images.');} catch (\Throwable $e) {+ report($e);+return response()->json(['message' => 'Failed to fetch Hetzner images.'], 500);}}@@ -422,6 +438,7 @@ class HetznerController extends Controllerresponse: 404,ref: '#/components/responses/404',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function sshKeys(Request $request)@@ -459,7 +476,11 @@ class HetznerController extends Controller$sshKeys = $hetznerService->getSshKeys();return response()->json($sshKeys);+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to fetch Hetzner SSH keys.');} catch (\Throwable $e) {+ report($e);+return response()->json(['message' => 'Failed to fetch Hetzner SSH keys.'], 500);}}@@ -517,6 +538,7 @@ class HetznerController extends Controllerresponse: 404,ref: '#/components/responses/404',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function firewalls(Request $request)@@ -553,7 +575,11 @@ class HetznerController extends Controller$hetznerService = new HetznerService($token->token);return response()->json($hetznerService->getFirewalls());+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to fetch Hetzner firewalls.');} catch (\Throwable $e) {+ report($e);+return response()->json(['message' => 'Failed to fetch Hetzner firewalls.'], 500);}}@@ -612,6 +638,7 @@ class HetznerController extends Controllerresponse: 404,ref: '#/components/responses/404',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function networks(Request $request)@@ -648,7 +675,11 @@ class HetznerController extends Controller$hetznerService = new HetznerService($token->token);return response()->json($hetznerService->getNetworks());+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to fetch Hetzner networks.');} catch (\Throwable $e) {+ report($e);+return response()->json(['message' => 'Failed to fetch Hetzner networks.'], 500);}}@@ -728,6 +759,7 @@ class HetznerController extends Controllerresponse: 429,ref: '#/components/responses/429',),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function createServer(Request $request)@@ -1012,7 +1044,18 @@ class HetznerController extends Controller}return $response;+ } catch (CloudProviderApiException $e) {+ return $e->toJsonResponse('Failed to create Hetzner server.');} catch (\Throwable $e) {+ report($e);+ if (isset($hetznerService, $hetznerServer['id'])) {+ try {+ $hetznerService->deleteServer((int) $hetznerServer['id']);+ } catch (\Throwable $cleanupError) {+ report($cleanupError);+ }+ }+return response()->json(['message' => 'Failed to create Hetzner server.'], 500);}}diff --git a/app/Http/Controllers/Api/OpenApi.php b/app/Http/Controllers/Api/OpenApi.phpindex 1279442b28..4af7e503bd 100644--- a/app/Http/Controllers/Api/OpenApi.php+++ b/app/Http/Controllers/Api/OpenApi.php@@ -198,6 +198,17 @@ use OpenApi\Attributes as OA;new OA\Property(property: 'message', type: 'string', example: 'Rate limit exceeded. Please try again later.'),])),+ new OA\Response(+ response: 502,+ description: 'The cloud provider API rejected the request.',+ content: new OA\JsonContent(+ type: 'object',+ properties: [+ new OA\Property(property: 'message', type: 'string', example: 'Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access.'),+ new OA\Property(property: 'provider_status', type: 'integer', description: 'The HTTP status from the cloud provider API.', example: 403),+ new OA\Property(property: 'provider_error_code', type: 'string', nullable: true, description: 'The error code from the cloud provider API, if it has one.', example: 'forbidden'),+ ]+ )),],)]class OpenApidiff --git a/app/Http/Controllers/Api/VultrController.php b/app/Http/Controllers/Api/VultrController.phpindex 31aa2b4bac..86601b8e4d 100644--- a/app/Http/Controllers/Api/VultrController.php+++ b/app/Http/Controllers/Api/VultrController.php@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;use App\Actions\Server\ValidateServer;use App\Enums\ProxyTypes;+use App\Exceptions\CloudProviderApiException;use App\Exceptions\RateLimitException;use App\Http\Controllers\Api\Concerns\RequiresDeployAbility;use App\Http\Controllers\Controller;@@ -193,6 +194,7 @@ class VultrController extends Controllernew OA\Response(response: 403, description: 'The token needs the deploy permission for a cloud-init script or extra SSH keys.'),new OA\Response(response: 422, description: 'Validation failed.'),new OA\Response(response: 429, description: 'Vultr API rate limit exceeded.'),+ new OA\Response(response: 502, ref: '#/components/responses/502'),])]public function createServer(Request $request): JsonResponse@@ -403,12 +405,13 @@ class VultrController extends Controller}return $response;- } catch (\Throwable $e) {+ } catch (CloudProviderApiException $e) {$this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server);- logger()->error('Failed to create Vultr server', [- 'error' => $e->getMessage(),- ]);+ return $e->toJsonResponse('Failed to create Vultr server.');+ } catch (\Throwable $e) {+ $this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server);+ report($e);return response()->json(['message' => 'Failed to create Vultr server.'], 500);}diff --git a/app/Livewire/Server/New/ByHetzner.php b/app/Livewire/Server/New/ByHetzner.phpindex 868b9e44b3..fbc6d93a9b 100644--- a/app/Livewire/Server/New/ByHetzner.php+++ b/app/Livewire/Server/New/ByHetzner.php@@ -3,6 +3,7 @@namespace App\Livewire\Server\New;use App\Enums\ProxyTypes;+use App\Exceptions\CloudProviderApiException;use App\Models\CloudInitScript;use App\Models\CloudProviderToken;use App\Models\PrivateKey;@@ -12,7 +13,6 @@ use App\Rules\ValidCloudInitYaml;use App\Rules\ValidHostname;use App\Services\HetznerService;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;-use Illuminate\Http\Client\RequestException;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Http;use Illuminate\Validation\ValidationException;@@ -358,22 +358,13 @@ class ByHetzner extends Component$this->loading_data = false;} catch (\Throwable $e) {$this->loading_data = false;- $this->provider_data_error = $this->providerDataErrorMessage('Hetzner', $e, 'error.message');+ $this->provider_data_error = $e instanceof CloudProviderApiException+ ? $e->getMessage()+ : 'Hetzner API error: '.$e->getMessage();$this->dispatch('error', $this->provider_data_error);}}- private function providerDataErrorMessage(string $providerName, \Throwable $e, string $jsonMessageKey): string- {- $details = $e->getMessage();-- if ($e instanceof RequestException && $e->response) {- $details = data_get($e->response->json(), $jsonMessageKey) ?: $e->response->body() ?: $details;- }-- return "{$providerName} API error: {$details}";- }-private function getCpuVendorInfo(array $serverType): ?string{$name = strtolower($serverType['name'] ?? '');@@ -766,20 +757,31 @@ class ByHetzner extends Componentif (! isset($e->errors()['server'])) {throw $e;}- if (isset($hetznerService, $hetznerServer['id'])) {- try {- $hetznerService->deleteServer((int) $hetznerServer['id']);- } catch (\Throwable $cleanupError) {- report($cleanupError);- }- }+ $this->deleteUntrackedHetznerServer($hetznerService ?? null, $hetznerServer['id'] ?? null);return $this->dispatch('error', 'You have reached the server limit for your subscription.');} catch (\Throwable $e) {+ if (! isset($server)) {+ $this->deleteUntrackedHetznerServer($hetznerService ?? null, $hetznerServer['id'] ?? null);+ }+return handleError($e, $this);}}+ private function deleteUntrackedHetznerServer(?HetznerService $hetznerService, ?int $hetznerServerId): void+ {+ if (! $hetznerService || ! $hetznerServerId) {+ return;+ }++ try {+ $hetznerService->deleteServer($hetznerServerId);+ } catch (\Throwable $e) {+ report($e);+ }+ }+public function render(){return view('livewire.server.new.by-hetzner');diff --git a/app/Services/DigitalOceanService.php b/app/Services/DigitalOceanService.phpindex da1af942f4..423fc4f0dc 100644--- a/app/Services/DigitalOceanService.php+++ b/app/Services/DigitalOceanService.php@@ -2,6 +2,7 @@namespace App\Services;+use App\Exceptions\CloudProviderApiException;use App\Exceptions\RateLimitException;use Illuminate\Http\Client\RequestException;use Illuminate\Support\Facades\Http;@@ -45,7 +46,7 @@ class DigitalOceanService);}- throw new \Exception('DigitalOcean API error: '.$response->json('message', 'Unknown error'), $response->status());+ throw new CloudProviderApiException('DigitalOcean', $response->json('message', 'Unknown error'), $response->status(), $response->json('id'));}return $response->json() ?? [];diff --git a/app/Services/HetznerService.php b/app/Services/HetznerService.phpindex c9675a24c8..faeeee654f 100644--- a/app/Services/HetznerService.php+++ b/app/Services/HetznerService.php@@ -2,6 +2,7 @@namespace App\Services;+use App\Exceptions\CloudProviderApiException;use App\Exceptions\RateLimitException;use Illuminate\Http\Client\RequestException;use Illuminate\Support\Facades\Http;@@ -44,7 +45,17 @@ class HetznerService// Exponential backoff for other retriable errors: 100ms, 200ms, 400msreturn $attempt * 100;- })+ }, function (\Exception $exception) use ($method) {+ if ($exception instanceof RequestException && $exception->response->status() === 429) {+ return true;+ }++ // A POST that failed with a 5xx or a timeout can still have created the resource, so only idempotent requests are sent again.+ // Client errors such as 401, 403, or 422 do not change when the request is sent again.+ $isServerError = ! $exception instanceof RequestException || $exception->response->status() >= 500;++ return $isServerError && in_array(strtolower($method), ['get', 'delete'], true);+ }, throw: false)->{$method}($this->baseUrl.$endpoint, $data);if (! $response->successful()) {@@ -61,7 +72,13 @@ class HetznerService);}- throw new \Exception('Hetzner API error: '.$response->json('error.message', 'Unknown error'));+ throw new CloudProviderApiException(+ 'Hetzner',+ $response->json('error.message', 'Unknown error'),+ $response->status(),+ $response->json('error.code'),+ collect($response->json('error.details.fields', []))->pluck('name')->all(),+ );}return $response->json();diff --git a/app/Services/VultrService.php b/app/Services/VultrService.phpindex 347b7e3fbb..ced66a66f7 100644--- a/app/Services/VultrService.php+++ b/app/Services/VultrService.php@@ -2,6 +2,7 @@namespace App\Services;+use App\Exceptions\CloudProviderApiException;use App\Exceptions\RateLimitException;use Illuminate\Support\Facades\Http;@@ -28,7 +29,7 @@ class VultrService);}- throw new \Exception('Vultr API error: '.$response->json('error', 'Unknown error'), $response->status());+ throw new CloudProviderApiException('Vultr', $response->json('error', 'Unknown error'), $response->status());}return $response->json() ?? [];diff --git a/openapi.json b/openapi.jsonindex 6b1ce726f5..2640e3d628 100644--- a/openapi.json+++ b/openapi.json@@ -11529,6 +11529,9 @@},"429": {"description": "DigitalOcean rate limit exceeded."+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -12567,6 +12570,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -12678,6 +12684,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -12758,6 +12767,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -12829,6 +12841,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -12894,6 +12909,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -12962,6 +12980,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -13122,6 +13143,9 @@},"429": {"$ref": "#\/components\/responses\/429"+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -24009,6 +24033,9 @@},"429": {"description": "Vultr API rate limit exceeded."+ },+ "502": {+ "$ref": "#\/components\/responses\/502"}},"security": [@@ -26425,6 +26452,35 @@}}}+ },+ "502": {+ "description": "The cloud provider API rejected the request.",+ "content": {+ "application\/json": {+ "schema": {+ "properties": {+ "message": {+ "type": "string",+ "example": "Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access."+ },+ "provider_status": {+ "description": "The HTTP status from the cloud provider API.",+ "type": "integer",+ "example": 403+ },+ "provider_error_code": {+ "description": "The error code from the cloud provider API, if it has one.",+ "type": [+ "string",+ "null"+ ],+ "example": "forbidden"+ }+ },+ "type": "object"+ }+ }+ }}},"securitySchemes": {diff --git a/openapi.yaml b/openapi.yamlindex 85f5c49369..2340bff7d9 100644--- a/openapi.yaml+++ b/openapi.yaml@@ -7536,6 +7536,8 @@ paths:description: 'Validation failed.''429':description: 'DigitalOcean rate limit exceeded.'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8117,6 +8119,8 @@ paths:$ref: '#/components/responses/401''404':$ref: '#/components/responses/404'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8157,6 +8161,8 @@ paths:$ref: '#/components/responses/401''404':$ref: '#/components/responses/404'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8197,6 +8203,8 @@ paths:$ref: '#/components/responses/401''404':$ref: '#/components/responses/404'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8237,6 +8245,8 @@ paths:$ref: '#/components/responses/401''404':$ref: '#/components/responses/404'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8277,6 +8287,8 @@ paths:$ref: '#/components/responses/401''404':$ref: '#/components/responses/404'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8317,6 +8329,8 @@ paths:$ref: '#/components/responses/401''404':$ref: '#/components/responses/404'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -8423,6 +8437,8 @@ paths:$ref: '#/components/responses/422''429':$ref: '#/components/responses/429'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -15197,6 +15213,8 @@ paths:description: 'Validation failed.''429':description: 'Vultr API rate limit exceeded.'+ '502':+ $ref: '#/components/responses/502'security:-bearerAuth: []@@ -16935,6 +16953,26 @@ components:type: stringexample: 'Rate limit exceeded. Please try again later.'type: object+ '502':+ description: 'The cloud provider API rejected the request.'+ content:+ application/json:+ schema:+ properties:+ message:+ type: string+ example: 'Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access.'+ provider_status:+ description: 'The HTTP status from the cloud provider API.'+ type: integer+ example: 403+ provider_error_code:+ description: 'The error code from the cloud provider API, if it has one.'+ type:+ - string+ - 'null'+ example: forbidden+ type: objectsecuritySchemes:bearerAuth:type: httpdiff --git a/tests/Feature/Api/HetznerApiTest.php b/tests/Feature/Api/HetznerApiTest.phpindex 4246eb7878..2916004e71 100644--- a/tests/Feature/Api/HetznerApiTest.php+++ b/tests/Feature/Api/HetznerApiTest.php@@ -808,8 +808,142 @@ describe('POST /api/v1/servers/hetzner', function () {});});+describe('POST /api/v1/servers/hetzner upstream errors', function () {+ beforeEach(function () {+ $this->createServerRequest = fn () => $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ 'Content-Type' => 'application/json',+ ])->postJson('/api/v1/servers/hetzner', [+ 'cloud_provider_token_id' => $this->hetznerToken->uuid,+ 'location' => 'nbg1',+ 'server_type' => 'cx11',+ 'image' => 15512617,+ 'name' => 'test-server',+ 'private_key_uuid' => $this->privateKey->uuid,+ 'enable_ipv4' => true,+ 'enable_ipv6' => true,+ ]);++ $this->fakeHetzner = fn (Closure $createServerResponse) => Http::fake(function (HttpRequest $request) use ($createServerResponse) {+ if ($request->method() === 'GET' && str_starts_with($request->url(), 'https://api.hetzner.cloud/v1/ssh_keys')) {+ return Http::response(['ssh_keys' => [], 'meta' => ['pagination' => ['next_page' => null]]], 200);+ }+ if ($request->method() === 'POST' && $request->url() === 'https://api.hetzner.cloud/v1/ssh_keys') {+ return Http::response(['ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd']], 201);+ }+ if ($request->method() === 'POST' && $request->url() === 'https://api.hetzner.cloud/v1/servers') {+ return $createServerResponse();+ }+ if ($request->method() === 'DELETE' && $request->url() === 'https://api.hetzner.cloud/v1/servers/456') {+ return Http::response([], 200);+ }++ return Http::response([], 404);+ });+ });++ test('explains a read-only token without leaking the Hetzner message and does not retry', function () {+ ($this->fakeHetzner)(fn () => Http::response([+ 'error' => ['code' => 'forbidden', 'message' => 'INTERNAL_LEAK_TOKEN_abc'],+ ], 403));++ $response = ($this->createServerRequest)();++ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access.',+ 'provider_status' => 403,+ 'provider_error_code' => 'forbidden',+ ]);+ expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');+ Http::assertSentCount(3);+ $this->assertDatabaseMissing('servers', ['name' => 'test-server']);+ });++ test('lists the invalid fields Hetzner reports', function () {+ ($this->fakeHetzner)(fn () => Http::response([+ 'error' => [+ 'code' => 'invalid_input',+ 'message' => 'invalid input in field server_type',+ 'details' => ['fields' => [['name' => 'server_type'], ['name' => '<script>']]],+ ],+ ], 422));++ $response = ($this->createServerRequest)();++ $response->assertStatus(502);+ $response->assertJson([+ 'message' => 'Failed to create Hetzner server. Hetzner rejected the request as invalid. Check the selected values and your account limits. Invalid fields: server_type.',+ 'provider_error_code' => 'invalid_input',+ ]);+ });++ test('does not send the create request again after a Hetzner server error', function () {+ ($this->fakeHetzner)(fn () => Http::response([+ 'error' => ['code' => 'unavailable', 'message' => 'service unavailable'],+ ], 503));++ $response = ($this->createServerRequest)();++ $response->assertStatus(502);+ $response->assertJson(['message' => 'Failed to create Hetzner server. Hetzner cannot process the request now. Try again later.']);+ Http::assertSentCount(3);+ });++ test('deletes the Hetzner server when registration in Coolify fails', function () {+ ($this->fakeHetzner)(fn () => Http::response([+ 'server' => ['id' => 456, 'name' => 'test-server', 'public_net' => []],+ ], 201));++ $response = ($this->createServerRequest)();++ $response->assertStatus(500);+ $response->assertExactJson(['message' => 'Failed to create Hetzner server.']);+ Http::assertSent(fn (HttpRequest $request) => $request->method() === 'DELETE'+ && $request->url() === 'https://api.hetzner.cloud/v1/servers/456');+ });+});++describe('GET /api/v1/hetzner list endpoints upstream errors', function () {+ test('explains an invalid Hetzner token and does not retry it', function () {+ Http::fake([+ 'https://api.hetzner.cloud/v1/locations*' => Http::response([+ 'error' => ['code' => 'unauthorized', 'message' => 'unable to authenticate'],+ ], 401),+ ]);++ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ ])->getJson('/api/v1/hetzner/locations?cloud_provider_token_id='.$this->hetznerToken->uuid);++ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner locations. The Hetzner API token is invalid or expired. Update the cloud provider token in Coolify.',+ 'provider_status' => 401,+ 'provider_error_code' => 'unauthorized',+ ]);+ Http::assertSentCount(1);+ });++ test('sends a read request again after a Hetzner server error', function () {+ Http::fake([+ 'https://api.hetzner.cloud/v1/locations*' => Http::sequence()+ ->push(['error' => ['code' => 'unavailable', 'message' => 'service unavailable']], 503)+ ->push(['locations' => [['id' => 1, 'name' => 'nbg1']], 'meta' => ['pagination' => ['next_page' => null]]], 200),+ ]);++ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ ])->getJson('/api/v1/hetzner/locations?cloud_provider_token_id='.$this->hetznerToken->uuid);++ $response->assertOk();+ $response->assertJsonFragment(['name' => 'nbg1']);+ Http::assertSentCount(2);+ });+});+describe('error responses do not leak exception details', function () {- test('locations endpoint returns generic 500 message on upstream failure', function () {+ test('locations endpoint returns a safe 502 message on upstream failure', function () {Http::fake(['https://api.hetzner.cloud/v1/locations*' => Http::response(['error' => ['message' => 'INTERNAL_LEAK_TOKEN_abc /var/secret/path'],@@ -821,13 +955,17 @@ describe('error responses do not leak exception details', function () {'Content-Type' => 'application/json',])->getJson('/api/v1/hetzner/locations?cloud_provider_token_id='.$this->hetznerToken->uuid);- $response->assertStatus(500);- $response->assertExactJson(['message' => 'Failed to fetch Hetzner locations.']);+ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner locations. Hetzner cannot process the request now. Try again later.',+ 'provider_status' => 500,+ 'provider_error_code' => null,+ ]);expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');expect($response->getContent())->not->toContain('/var/secret/path');});- test('server-types endpoint returns generic 500 message on upstream failure', function () {+ test('server-types endpoint returns a safe 502 message on upstream failure', function () {Http::fake(['https://api.hetzner.cloud/v1/server_types*' => Http::response(['error' => ['message' => 'INTERNAL_LEAK_TOKEN_abc'],@@ -839,12 +977,16 @@ describe('error responses do not leak exception details', function () {'Content-Type' => 'application/json',])->getJson('/api/v1/hetzner/server-types?cloud_provider_token_id='.$this->hetznerToken->uuid);- $response->assertStatus(500);- $response->assertExactJson(['message' => 'Failed to fetch Hetzner server types.']);+ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner server types. Hetzner cannot process the request now. Try again later.',+ 'provider_status' => 500,+ 'provider_error_code' => null,+ ]);expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');});- test('images endpoint returns generic 500 message on upstream failure', function () {+ test('images endpoint returns a safe 502 message on upstream failure', function () {Http::fake(['https://api.hetzner.cloud/v1/images*' => Http::response(['error' => ['message' => 'INTERNAL_LEAK_TOKEN_abc'],@@ -856,12 +998,16 @@ describe('error responses do not leak exception details', function () {'Content-Type' => 'application/json',])->getJson('/api/v1/hetzner/images?cloud_provider_token_id='.$this->hetznerToken->uuid);- $response->assertStatus(500);- $response->assertExactJson(['message' => 'Failed to fetch Hetzner images.']);+ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner images. Hetzner cannot process the request now. Try again later.',+ 'provider_status' => 500,+ 'provider_error_code' => null,+ ]);expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');});- test('ssh-keys endpoint returns generic 500 message on upstream failure', function () {+ test('ssh-keys endpoint returns a safe 502 message on upstream failure', function () {Http::fake(['https://api.hetzner.cloud/v1/ssh_keys*' => Http::response(['error' => ['message' => 'INTERNAL_LEAK_TOKEN_abc'],@@ -873,12 +1019,16 @@ describe('error responses do not leak exception details', function () {'Content-Type' => 'application/json',])->getJson('/api/v1/hetzner/ssh-keys?cloud_provider_token_id='.$this->hetznerToken->uuid);- $response->assertStatus(500);- $response->assertExactJson(['message' => 'Failed to fetch Hetzner SSH keys.']);+ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner SSH keys. Hetzner cannot process the request now. Try again later.',+ 'provider_status' => 500,+ 'provider_error_code' => null,+ ]);expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');});- test('firewalls endpoint returns generic 500 message on upstream failure', function () {+ test('firewalls endpoint returns a safe 502 message on upstream failure', function () {Http::fake(['https://api.hetzner.cloud/v1/firewalls*' => Http::response(['error' => ['message' => 'INTERNAL_LEAK_TOKEN_abc'],@@ -890,12 +1040,16 @@ describe('error responses do not leak exception details', function () {'Content-Type' => 'application/json',])->getJson('/api/v1/hetzner/firewalls?cloud_provider_token_id='.$this->hetznerToken->uuid);- $response->assertStatus(500);- $response->assertExactJson(['message' => 'Failed to fetch Hetzner firewalls.']);+ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner firewalls. Hetzner cannot process the request now. Try again later.',+ 'provider_status' => 500,+ 'provider_error_code' => null,+ ]);expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');});- test('networks endpoint returns generic 500 message on upstream failure', function () {+ test('networks endpoint returns a safe 502 message on upstream failure', function () {Http::fake(['https://api.hetzner.cloud/v1/networks*' => Http::response(['error' => ['message' => 'INTERNAL_LEAK_TOKEN_abc'],@@ -907,8 +1061,12 @@ describe('error responses do not leak exception details', function () {'Content-Type' => 'application/json',])->getJson('/api/v1/hetzner/networks?cloud_provider_token_id='.$this->hetznerToken->uuid);- $response->assertStatus(500);- $response->assertExactJson(['message' => 'Failed to fetch Hetzner networks.']);+ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to fetch Hetzner networks. Hetzner cannot process the request now. Try again later.',+ 'provider_status' => 500,+ 'provider_error_code' => null,+ ]);expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');});});diff --git a/tests/Feature/DigitalOceanApiTest.php b/tests/Feature/DigitalOceanApiTest.phpindex 8268999576..d7233b85cc 100644--- a/tests/Feature/DigitalOceanApiTest.php+++ b/tests/Feature/DigitalOceanApiTest.php@@ -7,6 +7,7 @@ use App\Models\Server;use App\Models\Team;use App\Models\User;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Exceptions;use Illuminate\Support\Facades\Http;use Illuminate\Support\Once;@@ -183,6 +184,43 @@ describe('POST /api/v1/servers/digitalocean', function () {]);});+ test('explains a DigitalOcean permission error without leaking the DigitalOcean message', function () {+ Http::fake([+ 'https://api.digitalocean.com/v2/account/keys' => Http::response([+ 'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],+ ], 201),+ 'https://api.digitalocean.com/v2/account/keys*' => Http::response([+ 'ssh_keys' => [],+ 'links' => ['pages' => []],+ ], 200),+ 'https://api.digitalocean.com/v2/droplets' => Http::response([+ 'id' => 'forbidden',+ 'message' => 'INTERNAL_LEAK_TOKEN_abc',+ ], 403),+ ]);++ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ 'Content-Type' => 'application/json',+ ])->postJson('/api/v1/servers/digitalocean', [+ 'cloud_provider_token_id' => $this->digitalOceanToken->uuid,+ 'region' => 'nyc1',+ 'size' => 's-1vcpu-1gb',+ 'image' => 'ubuntu-24-04-x64',+ 'name' => 'read-only-token',+ 'private_key_uuid' => $this->privateKey->uuid,+ ]);++ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to create DigitalOcean server. The DigitalOcean API token does not have permission for this request. Use a token with read and write access.',+ 'provider_status' => 403,+ 'provider_error_code' => 'forbidden',+ ]);+ expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');+ $this->assertDatabaseMissing('servers', ['name' => 'read-only-token']);+ });+test('deletes the droplet when local server persistence fails', function () {Http::fake(['https://api.digitalocean.com/v2/account/keys' => Http::response([@@ -207,6 +245,7 @@ describe('POST /api/v1/servers/digitalocean', function () {'https://api.digitalocean.com/v2/droplets/987' => Http::response(null, 204),]);+ Exceptions::fake();$eventDispatcher = Server::getEventDispatcher();Server::setEventDispatcher(clone $eventDispatcher);Server::created(function (): void {@@ -233,6 +272,7 @@ describe('POST /api/v1/servers/digitalocean', function () {$this->assertDatabaseMissing('servers', ['digitalocean_droplet_id' => 987,]);+ Exceptions::assertReported(fn (RuntimeException $e) => $e->getMessage() === 'local persistence failed');Http::assertSent(fn ($request) => $request->method() === 'DELETE'&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');});diff --git a/tests/Feature/Server/HetznerServerCreationTest.php b/tests/Feature/Server/HetznerServerCreationTest.phpindex d16e709cc7..2ccb23c643 100644--- a/tests/Feature/Server/HetznerServerCreationTest.php+++ b/tests/Feature/Server/HetznerServerCreationTest.php@@ -3,6 +3,7 @@use App\Livewire\Server\New\ByHetzner;use App\Models\CloudProviderToken;use App\Models\PrivateKey;+use App\Models\Server;use App\Models\Team;use App\Models\User;use Illuminate\Foundation\Testing\RefreshDatabase;@@ -219,7 +220,7 @@ describe('Boarding Flow Integration', function () {->assertSeeHtml('x-text="current">Select SSH keys</span>');});- test('shows the cloud init script name only when saving the script', function () {+ test('shows the cloud init script name only when saving the script', function () {Livewire::test(ByHetzner::class)->set('current_step', 2)->set('show_cloud_init_script', true)@@ -306,6 +307,44 @@ describe('Hetzner data loading', function () {->and($component->get('hetznerNetworks'))->toHaveCount(1);});+ test('deletes the Hetzner server when saving it in Coolify fails', function () {+ Http::fake([+ 'https://api.hetzner.cloud/v1/ssh_keys' => Http::response(['ssh_key' => ['id' => 123]], 201),+ 'https://api.hetzner.cloud/v1/ssh_keys*' => Http::response([+ 'ssh_keys' => [],+ 'meta' => ['pagination' => ['next_page' => null]],+ ], 200),+ 'https://api.hetzner.cloud/v1/servers' => Http::response([+ 'server' => ['id' => 456, 'public_net' => ['ipv4' => ['ip' => '203.0.113.10']]],+ ], 201),+ 'https://api.hetzner.cloud/v1/servers/456' => Http::response([], 200),+ ]);++ $eventDispatcher = Server::getEventDispatcher();+ Server::setEventDispatcher(clone $eventDispatcher);+ Server::created(function (): void {+ throw new RuntimeException('local persistence failed');+ });++ try {+ Livewire::test(ByHetzner::class)+ ->set('current_step', 2)+ ->set('selected_token_id', $this->hetznerToken->id)+ ->set('server_name', 'test-server')+ ->set('selected_location', 'nbg1')+ ->set('selected_server_type', 'cx11')+ ->set('selected_image', 15512617)+ ->set('private_key_id', $this->privateKey->id)+ ->call('submit');+ } finally {+ Server::setEventDispatcher($eventDispatcher);+ }++ $this->assertDatabaseMissing('servers', ['hetzner_server_id' => 456]);+ Http::assertSent(fn ($request) => $request->method() === 'DELETE'+ && $request->url() === 'https://api.hetzner.cloud/v1/servers/456');+ });+test('rejects submitting without a public IP protocol before calling Hetzner', function () {Http::fake();diff --git a/tests/Feature/VultrApiTest.php b/tests/Feature/VultrApiTest.phpindex e9c0ee24aa..85a421e60e 100644--- a/tests/Feature/VultrApiTest.php+++ b/tests/Feature/VultrApiTest.php@@ -10,6 +10,7 @@ use App\Models\User;use Illuminate\Auth\Access\AuthorizationException;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Http\Request;+use Illuminate\Support\Facades\Exceptions;use Illuminate\Support\Facades\Http;uses(RefreshDatabase::class);@@ -301,6 +302,43 @@ describe('POST /api/v1/servers/vultr', function () {]);});+ test('explains a Vultr permission error without leaking the Vultr message', function () {+ Http::fake([+ 'https://api.vultr.com/v2/ssh-keys' => Http::response([+ 'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],+ ], 201),+ 'https://api.vultr.com/v2/ssh-keys*' => Http::response([+ 'ssh_keys' => [],+ 'meta' => ['links' => ['next' => null]],+ ], 200),+ 'https://api.vultr.com/v2/instances' => Http::response([+ 'error' => 'INTERNAL_LEAK_TOKEN_abc',+ 'status' => 403,+ ], 403),+ ]);++ $response = $this->withHeaders([+ 'Authorization' => 'Bearer '.$this->bearerToken,+ 'Content-Type' => 'application/json',+ ])->postJson('/api/v1/servers/vultr', [+ 'cloud_provider_token_id' => $this->vultrToken->uuid,+ 'region' => 'ewr',+ 'plan' => 'vc2-1c-1gb',+ 'os_id' => 2284,+ 'name' => 'read-only-token',+ 'private_key_uuid' => $this->privateKey->uuid,+ ]);++ $response->assertStatus(502);+ $response->assertExactJson([+ 'message' => 'Failed to create Vultr server. The Vultr API token does not have permission for this request. Use a token with read and write access.',+ 'provider_status' => 403,+ 'provider_error_code' => null,+ ]);+ expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc');+ $this->assertDatabaseMissing('servers', ['name' => 'read-only-token']);+ });+test('deletes the Vultr instance when local server persistence fails', function () {Http::fake(['https://api.vultr.com/v2/ssh-keys' => Http::response([@@ -320,6 +358,7 @@ describe('POST /api/v1/servers/vultr', function () {'https://api.vultr.com/v2/instances/instance-persistence-fails' => Http::response(null, 204),]);+ Exceptions::fake();$eventDispatcher = Server::getEventDispatcher();Server::setEventDispatcher(clone $eventDispatcher);Server::created(function (): void {@@ -347,6 +386,7 @@ describe('POST /api/v1/servers/vultr', function () {$this->assertDatabaseMissing('servers', ['vultr_instance_id' => 'instance-persistence-fails',]);+ Exceptions::assertReported(fn (RuntimeException $e) => $e->getMessage() === 'local persistence failed');Http::assertSent(fn ($request) => $request->method() === 'DELETE'&& $request->url() === 'https://api.vultr.com/v2/instances/instance-persistence-fails');});