Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

feat(gpu): fix GPU compose settings and show detected server GPUs

Andras Bacsaiad6e929a933c
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php
index acc4d3e7b6..25aec367bc 100644
--- a/app/Jobs/ApplicationDeploymentJob.php
+++ b/app/Jobs/ApplicationDeploymentJob.php
@@ -13,6 +13,7 @@ use App\Exceptions\DeploymentException;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationPreview;
+use App\Models\ApplicationSetting;
use App\Models\EnvironmentVariable;
use App\Models\GithubApp;
use App\Models\GitlabApp;
@@ -4265,21 +4266,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
}
if ($this->application->settings->is_gpu_enabled) {
$docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [
- [
- 'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'),
- 'capabilities' => ['gpu'],
- 'options' => data_get($this->application, 'settings.gpu_options', []),
- ],
+ $this->application->settings->gpuDeviceReservation(),
];
- if (data_get($this->application, 'settings.gpu_count')) {
- $count = data_get($this->application, 'settings.gpu_count');
- if ($count === 'all') {
- $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = $count;
- } else {
- $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count;
- }
- } elseif (data_get($this->application, 'settings.gpu_device_ids')) {
- $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($this->application, 'settings.gpu_device_ids');
+ if (ApplicationSetting::parseGpuOptions($this->application->settings->gpu_options) === null) {
+ $this->application_deployment_queue->addLogEntry('GPU options are not a valid key=value list or YAML mapping. They are ignored.');
}
}
if ($this->application->isHealthcheckDisabled()) {
@@ -4309,6 +4299,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
if ($this->pull_request_id === 0) {
$custom_compose = convertDockerRunToCompose($this->application->custom_docker_run_options);
+ if ($this->application->settings->is_gpu_enabled && data_get($custom_compose, 'deploy.resources.reservations.devices')) {
+ data_forget($custom_compose, 'deploy.resources.reservations.devices');
+ $this->application_deployment_queue->addLogEntry('GPU access is enabled in the GPU settings, so --gpus in Custom Docker Options is ignored.');
+ }
if ((bool) $this->application->settings->is_consistent_container_name_enabled) {
$docker_compose['services'][$this->application->uuid] = $docker_compose['services'][$this->container_name];
if ($this->container_name !== $this->application->uuid) {
diff --git a/app/Livewire/Project/Application/Advanced.php b/app/Livewire/Project/Application/Advanced.php
index 86b406a6b5..0075bdddd4 100644
--- a/app/Livewire/Project/Application/Advanced.php
+++ b/app/Livewire/Project/Application/Advanced.php
@@ -260,11 +260,21 @@ class Advanced extends Component
{
try {
$this->authorize('update', $this->application);
- if ($this->gpuCount && $this->gpuDeviceIds) {
+ // Check only changed GPU values, so stored legacy values do not block other settings.
+ $settings = $this->application->settings;
+ $gpuSelectionChanged = $this->gpuCount !== $settings->gpu_count || $this->gpuDeviceIds !== $settings->gpu_device_ids;
+ if ($gpuSelectionChanged && filled($this->gpuCount) && filled($this->gpuDeviceIds)) {
$this->dispatch('error', 'You cannot set both GPU count and GPU device IDs.');
- $this->gpuCount = null;
- $this->gpuDeviceIds = null;
- $this->syncData(true);
+
+ return;
+ }
+ if ($this->gpuCount !== $settings->gpu_count && filled($this->gpuCount) && ! preg_match(ApplicationSetting::GPU_COUNT_PATTERN, trim($this->gpuCount))) {
+ $this->dispatch('error', 'GPU count must be a number or "all".');
+
+ return;
+ }
+ if ($this->gpuOptions !== $settings->gpu_options && ApplicationSetting::parseGpuOptions($this->gpuOptions) === null) {
+ $this->dispatch('error', 'GPU options must be key=value pairs (one per line) or a YAML mapping.');
return;
}
@@ -437,6 +447,33 @@ class Advanced extends Component
}
}
+ public function refreshGpus(): void
+ {
+ try {
+ $servers = $this->application->maintenanceServers();
+ foreach ($servers as $server) {
+ $this->authorize('update', $server);
+ }
+ $functionalServers = $servers->filter(fn ($server) => $server->isFunctional());
+ if ($functionalServers->isEmpty()) {
+ $this->dispatch('error', 'Validate the server connection before you refresh GPUs.');
+
+ return;
+ }
+
+ $failed = $functionalServers->reject(fn ($server) => $server->gatherServerMetadata())->pluck('name');
+ $this->application->unsetRelation('destination')->unsetRelation('additional_servers');
+ if ($failed->isNotEmpty()) {
+ $this->dispatch('error', 'Could not collect GPUs from '.$failed->implode(', ').'. Check the application logs for the remote command output.');
+
+ return;
+ }
+ $this->dispatch('success', 'GPUs refreshed.');
+ } catch (\Throwable $e) {
+ handleError($e, $this);
+ }
+ }
+
public function render()
{
$server = data_get($this->application, 'destination.server');
@@ -444,6 +481,7 @@ class Advanced extends Component
return view('livewire.project.application.advanced', [
'isTraefikProxy' => $server?->proxyType() === ProxyTypes::TRAEFIK->value,
'serverCertificateResolver' => $server?->traefikCertificateResolver() ?? ServerSetting::TRAEFIK_CERTIFICATE_RESOLVER_NONE,
+ 'gpuServers' => $this->application->maintenanceServers(),
]);
}
diff --git a/app/Models/ApplicationSetting.php b/app/Models/ApplicationSetting.php
index abe317530f..8ca5a0dbd3 100644
--- a/app/Models/ApplicationSetting.php
+++ b/app/Models/ApplicationSetting.php
@@ -5,6 +5,8 @@ namespace App\Models;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Model;
use OpenApi\Attributes as OA;
+use Symfony\Component\Yaml\Exception\ParseException;
+use Symfony\Component\Yaml\Yaml;
#[OA\Schema(
description: 'Application settings.',
@@ -59,6 +61,8 @@ class ApplicationSetting extends Model
*/
public const MAX_CONTAINER_NAME_PREFIX_LENGTH = 30;
+ public const GPU_COUNT_PATTERN = '/^(all|\d+)$/';
+
protected $casts = [
'is_static' => 'boolean',
'is_spa' => 'boolean',
@@ -170,6 +174,81 @@ class ApplicationSetting extends Model
return $this->stopGracePeriodSeconds();
}
+ /**
+ * The compose device request (deploy.resources.reservations.devices item) for the GPU settings.
+ * A count wins over device IDs. Options that cannot be parsed are left out.
+ *
+ * @return array{capabilities: list<string>, driver?: string, count?: int|string, device_ids?: list<string>, options?: array<string, string>}
+ */
+ public function gpuDeviceReservation(): array
+ {
+ $device = ['capabilities' => ['gpu']];
+ $driver = trim((string) $this->gpu_driver);
+ if ($driver !== '') {
+ $device['driver'] = $driver;
+ }
+
+ $count = trim((string) $this->gpu_count);
+ $deviceIds = str((string) $this->gpu_device_ids)->explode(',')->map(fn (string $id) => trim($id))->filter(fn (string $id) => $id !== '')->values()->all();
+ if ($count !== '') {
+ $device['count'] = $count === 'all' ? 'all' : (int) $count;
+ } elseif ($deviceIds !== []) {
+ $device['device_ids'] = $deviceIds;
+ }
+
+ $options = self::parseGpuOptions($this->gpu_options);
+ if ($options) {
+ $device['options'] = $options;
+ }
+
+ return $device;
+ }
+
+ /**
+ * Parse GPU driver options written as a YAML/JSON mapping or as key=value pairs, one per line.
+ *
+ * @return array<string, string>|null null when the value is not a flat mapping
+ */
+ public static function parseGpuOptions(?string $options): ?array
+ {
+ $options = trim((string) $options);
+ if ($options === '') {
+ return [];
+ }
+
+ try {
+ $parsed = Yaml::parse($options);
+ } catch (ParseException) {
+ $parsed = null;
+ }
+ if (is_array($parsed) && $parsed !== [] && ! array_is_list($parsed)) {
+ $mapping = [];
+ foreach ($parsed as $key => $value) {
+ if (! is_scalar($value)) {
+ return null;
+ }
+ $mapping[(string) $key] = is_bool($value) ? ($value ? 'true' : 'false') : (string) $value;
+ }
+
+ return $mapping;
+ }
+
+ $mapping = [];
+ foreach (preg_split('/\R/', $options) as $pair) {
+ $pair = trim($pair);
+ if ($pair === '') {
+ continue;
+ }
+ [$key, $value] = array_pad(explode('=', $pair, 2), 2, null);
+ if ($value === null || trim($key) === '') {
+ return null;
+ }
+ $mapping[trim($key)] = trim($value);
+ }
+
+ return $mapping;
+ }
+
public function isStatic(): Attribute
{
return Attribute::make(
diff --git a/app/Models/Server.php b/app/Models/Server.php
index c1cebd7287..00b10235b3 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -2102,7 +2102,7 @@ $siteAddress {
try {
$output = instant_remote_process([
- 'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true)',
+ 'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true) && echo "---GPUS---" && (nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits 2>/dev/null || true) && echo "---NVIDIA_TOOLKIT---" && (command -v nvidia-container-runtime-hook nvidia-ctk 2>/dev/null || true)',
], $this, false);
if (! $output) {
@@ -2110,6 +2110,7 @@ $siteAddress {
}
$sections = [];
+ $sectionLines = [];
$currentKey = null;
foreach (explode("\n", trim($output)) as $line) {
$line = trim($line);
@@ -2117,6 +2118,9 @@ $siteAddress {
$currentKey = $m[1];
} elseif ($currentKey) {
$sections[$currentKey] = $line;
+ if ($line !== '') {
+ $sectionLines[$currentKey][] = $line;
+ }
}
}
@@ -2127,6 +2131,8 @@ $siteAddress {
'cpus' => (int) ($sections['CPUS'] ?? 0),
'memory_bytes' => (int) ($sections['MEMORY'] ?? 0),
'uptime_since' => $sections['UPTIME_SINCE'] ?? null,
+ 'gpus' => self::parseNvidiaSmiGpus($sectionLines['GPUS'] ?? []),
+ 'nvidia_container_toolkit' => ($sectionLines['NVIDIA_TOOLKIT'] ?? []) !== [],
'collected_at' => now()->toIso8601String(),
];
@@ -2166,6 +2172,38 @@ $siteAddress {
}
}
+ /**
+ * Parse `nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits` lines.
+ * The name is the last column because it can contain commas.
+ *
+ * @param list<string> $lines
+ * @return list<array{index: string, uuid: string, memory_mb: int, name: string}>
+ */
+ public static function parseNvidiaSmiGpus(array $lines): array
+ {
+ $gpus = [];
+ foreach ($lines as $line) {
+ $columns = array_map('trim', explode(',', $line, 4));
+ if (count($columns) !== 4 || ! ctype_digit($columns[0])) {
+ continue;
+ }
+ [$index, $uuid, $memory, $name] = $columns;
+ $gpus[] = ['index' => $index, 'uuid' => $uuid, 'memory_mb' => (int) $memory, 'name' => $name];
+ }
+
+ return $gpus;
+ }
+
+ /**
+ * GPUs found when the server details were last collected, or null when they were never checked.
+ *
+ * @return list<array{index: string, uuid: string, memory_mb: int, name: string}>|null
+ */
+ public function detectedGpus(): ?array
+ {
+ return data_get($this->server_metadata, 'gpus');
+ }
+
public function isTerminalEnabled()
{
return $this->settings->is_terminal_enabled ?? false;
diff --git a/bootstrap/helpers/api.php b/bootstrap/helpers/api.php
index ca356fa16a..cebeef2d12 100644
--- a/bootstrap/helpers/api.php
+++ b/bootstrap/helpers/api.php
@@ -212,9 +212,13 @@ function sharedDataApplications(?Application $application = null): array
'is_log_drain_enabled' => 'boolean',
'is_gpu_enabled' => 'boolean',
'gpu_driver' => 'string|nullable',
- 'gpu_count' => 'string|nullable',
+ 'gpu_count' => ['string', 'nullable', 'regex:'.ApplicationSetting::GPU_COUNT_PATTERN],
'gpu_device_ids' => 'string|nullable',
- 'gpu_options' => 'string|nullable',
+ 'gpu_options' => ['string', 'nullable', function (string $attribute, mixed $value, Closure $fail): void {
+ if (ApplicationSetting::parseGpuOptions($value) === null) {
+ $fail('The gpu_options must be key=value pairs or a YAML mapping.');
+ }
+ }],
'is_consistent_container_name_enabled' => 'boolean',
'custom_internal_name' => ['nullable', ...ValidationPatterns::containerNameRules()],
'custom_container_name_prefix' => 'string|nullable|max:'.ApplicationSetting::MAX_CONTAINER_NAME_PREFIX_LENGTH,
diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php
index 33dd277d03..e4548c0aed 100644
--- a/bootstrap/helpers/docker.php
+++ b/bootstrap/helpers/docker.php
@@ -1641,11 +1641,17 @@ function convertDockerRunToCompose(?string $custom_docker_run_options = null)
foreach ($matches as $match) {
$option = $match[1];
if ($option === '--gpus') {
- $regexForParsingDeviceIds = '/device=([0-9A-Za-z-,]+)/';
- preg_match($regexForParsingDeviceIds, $custom_docker_run_options, $device_matches);
- $value = $device_matches[1] ?? 'all';
- $options[$option][] = $value;
- $options[$option] = array_unique($options[$option]);
+ // --gpus all | --gpus 2 | --gpus "device=0,1"
+ $gpuRequest = [];
+ $gpuValue = trim($match[2] ?? '', '\'"');
+ if (preg_match('/device=([0-9A-Za-z-,]+)/', $custom_docker_run_options, $device_matches)) {
+ $gpuRequest['device_ids'] = explode(',', $device_matches[1]);
+ } elseif (ctype_digit($gpuValue)) {
+ $gpuRequest['count'] = (int) $gpuValue;
+ }
+ $options[$option] = $gpuRequest;
+
+ continue;
}
if ($option === '--hostname') {
// Match --hostname=value or --hostname value
@@ -1736,23 +1742,14 @@ function convertDockerRunToCompose(?string $custom_docker_run_options = null)
$compose_options->put($mapping[$option], $value[0]);
}
} elseif ($option === '--gpus') {
- $payload = [
- 'driver' => 'nvidia',
- 'capabilities' => ['gpu'],
- ];
- if (! is_null($value) && is_array($value) && count($value) > 0 && ! empty(trim($value[0]))) {
- if (str($value[0]) != 'all') {
- if (str($value[0])->contains(',')) {
- $payload['device_ids'] = str($value[0])->explode(',')->toArray();
- } else {
- $payload['device_ids'] = [$value[0]];
- }
- }
- }
$compose_options->put('deploy', [
'resources' => [
'reservations' => [
- 'devices' => [$payload],
+ 'devices' => [[
+ 'driver' => 'nvidia',
+ 'capabilities' => ['gpu'],
+ ...$value,
+ ]],
],
],
]);
diff --git a/resources/views/components/server/gpu-list.blade.php b/resources/views/components/server/gpu-list.blade.php
new file mode 100644
index 0000000000..93c084dec5
--- /dev/null
+++ b/resources/views/components/server/gpu-list.blade.php
@@ -0,0 +1,28 @@
+@props(['server'])
+
+@php($gpus = $server->detectedGpus())
+
+<div {{ $attributes->merge(['class' => 'text-[12px] leading-5 text-neutral-600 dark:text-fg-dim']) }}>
+ @if ($gpus === null)
+ <p>Not checked yet. Refresh to find GPUs.</p>
+ @elseif ($gpus === [])
+ <p>No NVIDIA GPU found (<span class="font-mono">nvidia-smi</span> is not installed or shows no GPU).</p>
+ @else
+ <ul>
+ @foreach ($gpus as $gpu)
+ <li wire:key="server-{{ $server->id }}-gpu-{{ $gpu['uuid'] }}">
+ <span class="font-mono font-medium text-neutral-950 dark:text-fg">{{ $gpu['index'] }}</span>
+ · {{ $gpu['name'] }} · {{ round($gpu['memory_mb'] / 1024, 1) }} GB ·
+ <span class="font-mono">{{ $gpu['uuid'] }}</span>
+ </li>
+ @endforeach
+ </ul>
+ @unless (data_get($server->server_metadata, 'nvidia_container_toolkit'))
+ <p class="mt-1 text-warning-900 dark:text-warning">
+ NVIDIA Container Toolkit not found. Containers cannot use these GPUs until you
+ <a href="https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html"
+ class="underline" target="_blank">install it</a> and restart Docker.
+ </p>
+ @endunless
+ @endif
+</div>
diff --git a/resources/views/livewire/project/application/advanced.blade.php b/resources/views/livewire/project/application/advanced.blade.php
index c5e5aa89c3..9ce4bc1ae9 100644
--- a/resources/views/livewire/project/application/advanced.blade.php
+++ b/resources/views/livewire/project/application/advanced.blade.php
@@ -203,6 +203,30 @@
@if ($application->build_pack !== 'dockercompose')
<x-application.settings-section id="advanced-gpu-section" title="GPU"
helper="Give this application access to the host's GPUs. More info <a href='https://docs.docker.com/compose/gpu-support/' class='underline dark:text-white' target='_blank'>here</a>.">
+ <x-slot:actions>
+ @php($canRefreshGpus = $gpuServers->contains(fn ($gpuServer) => $gpuServer->isFunctional()))
+ <x-forms.button type="button" wire:click="refreshGpus" canGate="update"
+ :canResource="$gpuServers->first()" :disabled="! $canRefreshGpus"
+ :tooltip="$canRefreshGpus ? null : 'Validate the server connection first.'">
+ <x-reicon name="refresh" class="size-3.5" />
+ Refresh GPUs
+ </x-forms.button>
+ </x-slot:actions>
+ <x-callout type="info" title="Detected GPUs" class="mb-5">
+ @foreach ($gpuServers as $gpuServer)
+ <div wire:key="gpu-server-{{ $gpuServer->id }}" @class(['mt-2' => ! $loop->first])>
+ <p class="font-medium text-neutral-700 dark:text-fg">
+ <a href="{{ route('server.show', ['server_uuid' => $gpuServer->uuid]) }}"
+ class="underline">{{ $gpuServer->name }}</a>
+ @if (data_get($gpuServer->server_metadata, 'collected_at'))
+ <span class="font-normal text-neutral-500 dark:text-fg-dim">· checked
+ {{ \Illuminate\Support\Carbon::parse(data_get($gpuServer->server_metadata, 'collected_at'))->diffForHumans() }}</span>
+ @endif
+ </p>
+ <x-server.gpu-list :server="$gpuServer" />
+ </div>
+ @endforeach
+ </x-callout>
<div class="grid w-full gap-4 sm:grid-cols-2">
<x-forms.listbox id="isGpuEnabled" label="GPU access" onChange="instantSave"
:options="[
@@ -218,13 +242,14 @@
targets="gpuDriver,gpuCount,gpuDeviceIds,gpuOptions" />
<div class="grid gap-4 sm:grid-cols-2">
<x-forms.input label="GPU driver" id="gpuDriver" canGate="update" :canResource="$application" />
- <x-forms.input label="GPU count" placeholder="Empty means use all GPUs" id="gpuCount"
+ <x-forms.input label="GPU count" placeholder="Empty means use all GPUs" helper="A number, or all." id="gpuCount"
canGate="update" :canResource="$application" />
</div>
<x-forms.input label="GPU device ids" placeholder="0,2"
helper="Comma separated list of device ids. More info <a href='https://docs.docker.com/compose/gpu-support/#access-specific-devices' class='underline dark:text-white' target='_blank'>here</a>."
id="gpuDeviceIds" canGate="update" :canResource="$application" />
- <x-forms.textarea rows="6" label="GPU options" id="gpuOptions" canGate="update"
+ <x-forms.textarea rows="6" label="GPU options" id="gpuOptions" placeholder="key=value"
+ helper="Driver options. Use one key=value pair per line, or a YAML mapping." canGate="update"
:canResource="$application" />
</form>
@endif
diff --git a/resources/views/livewire/server/partials/server-details.blade.php b/resources/views/livewire/server/partials/server-details.blade.php
index be8d0236a0..262e54570d 100644
--- a/resources/views/livewire/server/partials/server-details.blade.php
+++ b/resources/views/livewire/server/partials/server-details.blade.php
@@ -21,3 +21,7 @@
</div>
@endforeach
</dl>
+<div class="mt-5">
+ <p class="text-xs font-medium text-neutral-500 dark:text-fg-dim">GPUs</p>
+ <x-server.gpu-list :server="$server" class="mt-2" />
+</div>
diff --git a/tests/Feature/ApplicationGpuSettingsTest.php b/tests/Feature/ApplicationGpuSettingsTest.php
new file mode 100644
index 0000000000..0f32b3f8b2
--- /dev/null
+++ b/tests/Feature/ApplicationGpuSettingsTest.php
@@ -0,0 +1,175 @@
+<?php
+
+use App\Livewire\Project\Application\Advanced;
+use App\Models\Application;
+use App\Models\ApplicationSetting;
+use App\Models\Environment;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Process;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+function createApplicationForGpuSettingsTest(): Application
+{
+ $team = Team::factory()->create();
+ $team->members()->attach(auth()->id(), ['role' => 'owner']);
+ session(['currentTeam' => $team]);
+ $server = Server::factory()->create(['team_id' => $team->id]);
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+ return Application::create([
+ 'name' => 'gpu-settings-test-app',
+ 'git_repository' => 'https://github.com/coollabsio/coolify',
+ 'git_branch' => 'main',
+ 'build_pack' => 'nixpacks',
+ 'ports_exposes' => '3000',
+ 'environment_id' => $environment->id,
+ 'destination_id' => $server->standaloneDockers()->firstOrFail()->id,
+ 'destination_type' => $server->standaloneDockers()->firstOrFail()->getMorphClass(),
+ ]);
+}
+
+it('builds a valid compose device request from the GPU settings', function (array $settings, array $expected) {
+ $setting = new ApplicationSetting(['gpu_driver' => 'nvidia', ...$settings]);
+
+ expect($setting->gpuDeviceReservation())->toBe($expected);
+})->with([
+ 'all gpus' => [[], ['capabilities' => ['gpu'], 'driver' => 'nvidia']],
+ 'count' => [['gpu_count' => '2'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 2]],
+ 'count all' => [['gpu_count' => 'all'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 'all']],
+ 'device ids' => [['gpu_device_ids' => '0, 2,'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'device_ids' => ['0', '2']]],
+ 'count wins over device ids' => [['gpu_count' => '1', 'gpu_device_ids' => '0'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 1]],
+ 'blank driver' => [['gpu_driver' => ''], ['capabilities' => ['gpu']]],
+ 'key=value options' => [['gpu_options' => "virtualization=false\nfoo = bar"], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['virtualization' => 'false', 'foo' => 'bar']]],
+ 'option value with commas' => [['gpu_options' => 'capabilities=compute,utility'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['capabilities' => 'compute,utility']]],
+ 'yaml options' => [['gpu_options' => "virtualization: false\nlevel: 2"], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['virtualization' => 'false', 'level' => '2']]],
+ 'empty options' => [['gpu_options' => ''], ['capabilities' => ['gpu'], 'driver' => 'nvidia']],
+ 'invalid options are left out' => [['gpu_options' => 'not an option'], ['capabilities' => ['gpu'], 'driver' => 'nvidia']],
+]);
+
+describe('advanced GPU form', function () {
+ beforeEach(function () {
+ $this->actingAs(User::factory()->create());
+ });
+
+ it('saves valid GPU options', function () {
+ $application = createApplicationForGpuSettingsTest();
+
+ Livewire::test(Advanced::class, ['application' => $application])
+ ->set('gpuCount', '2')
+ ->set('gpuOptions', 'virtualization=false')
+ ->call('submit')
+ ->assertDispatched('success');
+
+ $settings = $application->settings()->first();
+ expect($settings->gpu_count)->toBe('2')
+ ->and($settings->gpu_options)->toBe('virtualization=false');
+ });
+
+ it('rejects invalid GPU values', function (string $property, string $value) {
+ $application = createApplicationForGpuSettingsTest();
+
+ Livewire::test(Advanced::class, ['application' => $application])
+ ->set($property, $value)
+ ->call('submit')
+ ->assertDispatched('error')
+ ->assertNotDispatched('success');
+ })->with([
+ 'count' => ['gpuCount', 'two'],
+ 'options' => ['gpuOptions', 'not an option'],
+ ]);
+
+ it('does not block other settings when a stored GPU value is invalid', function () {
+ $application = createApplicationForGpuSettingsTest();
+ $application->settings->update(['gpu_options' => 'legacy value', 'gpu_count' => 'some']);
+
+ Livewire::test(Advanced::class, ['application' => $application->fresh()])
+ ->set('includeSourceCommitInBuild', true)
+ ->call('submit')
+ ->assertDispatched('success');
+
+ expect($application->settings()->first()->include_source_commit_in_build)->toBeTrue();
+ });
+
+ it('keeps a stored GPU count and device IDs pair when other settings are saved', function () {
+ $application = createApplicationForGpuSettingsTest();
+ $application->settings->update(['gpu_count' => '1', 'gpu_device_ids' => '0']);
+
+ Livewire::test(Advanced::class, ['application' => $application->fresh()])
+ ->set('includeSourceCommitInBuild', true)
+ ->call('submit')
+ ->assertDispatched('success');
+
+ $settings = $application->settings()->first();
+ expect($settings->gpu_count)->toBe('1')
+ ->and($settings->gpu_device_ids)->toBe('0')
+ ->and($settings->include_source_commit_in_build)->toBeTrue();
+ });
+
+ it('refreshes the GPUs of the application server', function () {
+ $application = createApplicationForGpuSettingsTest();
+ $server = $application->destination->server;
+ $server->update(['private_key_id' => PrivateKey::factory()->create(['team_id' => $server->team_id])->id]);
+ $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+ Process::fake(['*' => Process::result(output: "---GPUS---\n0, GPU-1a2b, 81559, NVIDIA H100\n---NVIDIA_TOOLKIT---\n/usr/bin/nvidia-ctk")]);
+
+ Livewire::test(Advanced::class, ['application' => $application])
+ ->call('refreshGpus')
+ ->assertDispatched('success')
+ ->assertSee('NVIDIA H100');
+
+ expect($server->fresh()->detectedGpus())->toBe([['index' => '0', 'uuid' => 'GPU-1a2b', 'memory_mb' => 81559, 'name' => 'NVIDIA H100']]);
+ });
+
+ it('does not refresh GPUs when no server is validated', function () {
+ $application = createApplicationForGpuSettingsTest();
+ $application->destination->server->settings->update(['is_reachable' => false, 'is_usable' => false]);
+ Process::fake();
+
+ Livewire::test(Advanced::class, ['application' => $application])
+ ->call('refreshGpus')
+ ->assertDispatched('error')
+ ->assertNotDispatched('success');
+
+ Process::assertNothingRan();
+ });
+
+ it('does not let a team member refresh GPUs', function () {
+ $application = createApplicationForGpuSettingsTest();
+ $server = $application->destination->server;
+ $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+ $member = User::factory()->create();
+ $application->team()->members()->attach($member->id, ['role' => 'member']);
+ $this->actingAs($member);
+ Process::fake();
+
+ Livewire::test(Advanced::class, ['application' => $application])
+ ->call('refreshGpus')
+ ->assertNotDispatched('success');
+
+ Process::assertNothingRan();
+ expect($server->fresh()->detectedGpus())->toBeNull();
+ });
+
+ it('rejects a change that sets both GPU count and device IDs without clearing stored values', function () {
+ $application = createApplicationForGpuSettingsTest();
+ $application->settings->update(['gpu_device_ids' => '0']);
+
+ Livewire::test(Advanced::class, ['application' => $application->fresh()])
+ ->set('gpuCount', '2')
+ ->call('submit')
+ ->assertDispatched('error')
+ ->assertNotDispatched('success');
+
+ $settings = $application->settings()->first();
+ expect($settings->gpu_count)->toBeNull()
+ ->and($settings->gpu_device_ids)->toBe('0');
+ });
+});
diff --git a/tests/Feature/Helpers/DockerCustomCommandsTest.php b/tests/Feature/Helpers/DockerCustomCommandsTest.php
index 299709ffd9..cf2408faf3 100644
--- a/tests/Feature/Helpers/DockerCustomCommandsTest.php
+++ b/tests/Feature/Helpers/DockerCustomCommandsTest.php
@@ -233,3 +233,22 @@ test('ConvertIpAndIp6Together', function () {
'ip6' => ['2001:db8::1'],
]);
});
+
+test('ConvertGpusCount', function () {
+ $output = convertDockerRunToCompose('--gpus 2');
+ expect($output)->toBe([
+ 'deploy' => [
+ 'resources' => [
+ 'reservations' => [
+ 'devices' => [
+ [
+ 'driver' => 'nvidia',
+ 'capabilities' => ['gpu'],
+ 'count' => 2,
+ ],
+ ],
+ ],
+ ],
+ ],
+ ]);
+});
diff --git a/tests/Feature/ServerMetadataTest.php b/tests/Feature/ServerMetadataTest.php
index ec33d4f0e8..5d521a71e6 100644
--- a/tests/Feature/ServerMetadataTest.php
+++ b/tests/Feature/ServerMetadataTest.php
@@ -220,3 +220,57 @@ it('does not call gatherServerMetadata when docker version validation fails', fu
->instance()
->validateDockerVersion();
});
+
+it('stores detected NVIDIA GPUs and the container toolkit when gathering server metadata', function (array $gpuLines, array $toolkitLines, array $expectedGpus, bool $expectedToolkit) {
+ $this->server->settings->update([
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ ]);
+
+ Process::fake([
+ '*' => Process::result(output: implode("\n", [
+ '---PRETTY_NAME---',
+ 'Ubuntu 24.04 LTS',
+ '---CPUS---',
+ '8',
+ '---DOCKER---',
+ '29.4.3-ce',
+ '---COMPOSE---',
+ 'v2.32.4',
+ '---GPUS---',
+ ...$gpuLines,
+ '---NVIDIA_TOOLKIT---',
+ ...$toolkitLines,
+ ]), exitCode: 0),
+ ]);
+
+ $this->server->gatherServerMetadata();
+ $server = $this->server->fresh();
+
+ expect($server->detectedGpus())->toBe($expectedGpus)
+ ->and($server->server_metadata['nvidia_container_toolkit'])->toBe($expectedToolkit)
+ ->and($server->server_metadata['cpus'])->toBe(8);
+})->with([
+ 'two gpus' => [
+ ['0, GPU-1a2b, 81559, NVIDIA H100 80GB HBM3', '1, GPU-3c4d, 24564, NVIDIA GeForce RTX 4090, Founders'],
+ ['/usr/bin/nvidia-container-runtime-hook', '/usr/bin/nvidia-ctk'],
+ [
+ ['index' => '0', 'uuid' => 'GPU-1a2b', 'memory_mb' => 81559, 'name' => 'NVIDIA H100 80GB HBM3'],
+ ['index' => '1', 'uuid' => 'GPU-3c4d', 'memory_mb' => 24564, 'name' => 'NVIDIA GeForce RTX 4090, Founders'],
+ ],
+ true,
+ ],
+ 'driver installed without a working gpu' => [
+ ['NVIDIA-SMI has failed because it could not communicate with the NVIDIA driver.'],
+ [],
+ [],
+ false,
+ ],
+ 'no nvidia-smi' => [[], [], [], false],
+]);
+
+it('returns null detected GPUs when GPUs were never checked', function () {
+ $this->server->update(['server_metadata' => ['os' => 'Ubuntu 24.04 LTS']]);
+
+ expect($this->server->fresh()->detectedGpus())->toBeNull();
+});