Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

feat(dev): pass the host NVIDIA GPU through to the KVM dev VM

Andras Bacsai0eb78dcf9b34
diff --git a/AGENTS.md b/AGENTS.md
index 661a920c56..bfa80f40cf 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -29,7 +29,7 @@ Docker Compose-based dev setup with services: coolify (app, which also runs Reve
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
```
-The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.
+The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. To pass the host NVIDIA GPU through to the KVM localhost VM (`ubuntu-root` only), start with `COOLIFY_DEV_GPU=1 ./scripts/dev start` (or `php artisan dev:qemu ubuntu-root --as-localhost --gpu`). This attaches every PCI function of the GPU, gives the VM 4 GiB, stops `nvidia-persistenced` on the host, and installs the NVIDIA driver and container toolkit in the VM. The host cannot use the GPU while the VM runs. Remove it with `--fresh` without `--gpu`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.
## Testing the Self-Hosted Upgrade Process
diff --git a/app/Actions/Development/InstallDevelopmentGpuDriver.php b/app/Actions/Development/InstallDevelopmentGpuDriver.php
new file mode 100644
index 0000000000..ffe62f497c
--- /dev/null
+++ b/app/Actions/Development/InstallDevelopmentGpuDriver.php
@@ -0,0 +1,47 @@
+<?php
+
+namespace App\Actions\Development;
+
+use App\Models\Server;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+/**
+ * Installs the NVIDIA driver and the NVIDIA Container Toolkit on an Ubuntu development VM with a passed-through GPU,
+ * then refreshes the server details so Coolify shows the GPU. Steps that are already done are skipped.
+ *
+ * @see https://ubuntu.com/server/docs/how-to/graphics/install-nvidia-drivers/
+ * @see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
+ */
+class InstallDevelopmentGpuDriver
+{
+ use AsAction;
+
+ public function handle(Server $server): void
+ {
+ instant_remote_process([
+ 'set -e',
+ 'export DEBIAN_FRONTEND=noninteractive',
+ 'if ! nvidia-smi >/dev/null 2>&1; then',
+ 'apt-get update -qq',
+ 'apt-get install -y -qq ubuntu-drivers-common',
+ 'ubuntu-drivers install --gpgpu',
+ // For example 580-server-open. The utilities package has no -open suffix.
+ 'flavor=$(dpkg-query -W -f=\'${Package}\n\' \'nvidia-headless-no-dkms-*\' | sed -n \'s/^nvidia-headless-no-dkms-//p\' | head -n1)',
+ // ubuntu-drivers installs the modules for the newest kernel, which can differ from the running one.
+ 'apt-get install -y -qq "nvidia-utils-${flavor%-open}" "linux-modules-nvidia-${flavor}-$(uname -r)"',
+ 'modprobe nvidia',
+ 'nvidia-smi',
+ 'fi',
+ 'if ! command -v nvidia-ctk >/dev/null 2>&1; then',
+ 'curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --batch --yes --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg',
+ 'curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed \'s#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g\' > /etc/apt/sources.list.d/nvidia-container-toolkit.list',
+ 'apt-get update -qq',
+ 'apt-get install -y -qq nvidia-container-toolkit',
+ 'nvidia-ctk runtime configure --runtime=docker',
+ 'systemctl restart docker',
+ 'fi',
+ ], $server, timeout: 1800, disableMultiplexing: true, maxAttempts: 1);
+
+ $server->gatherServerMetadata();
+ }
+}
diff --git a/app/Actions/Development/ManageDevelopmentQemuVm.php b/app/Actions/Development/ManageDevelopmentQemuVm.php
index 9b1a1c791f..9fab0050a4 100644
--- a/app/Actions/Development/ManageDevelopmentQemuVm.php
+++ b/app/Actions/Development/ManageDevelopmentQemuVm.php
@@ -2,6 +2,7 @@
namespace App\Actions\Development;
+use Illuminate\Contracts\Process\ProcessResult;
use Illuminate\Database\QueryException;
use Illuminate\Support\Facades\Process;
use Lorisleiva\Actions\Concerns\AsAction;
@@ -12,28 +13,44 @@ class ManageDevelopmentQemuVm
/**
* Start (or create) the selected VMs and seed their servers. Existing VMs are reused unless $fresh is set.
+ * With $gpu, the host GPU is passed through and the container seeds the server, because it installs the
+ * driver over Coolify's SSH connection.
*
* @param string|array<int, string> $profileNames
*/
- public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false): void
+ public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false, bool $gpu = false): void
{
$profileNames = is_array($profileNames) ? array_values(array_unique($profileNames)) : [$profileNames];
foreach ($profileNames as $index => $profileName) {
- StartDevelopmentQemuVm::run($profileName, $fresh);
+ StartDevelopmentQemuVm::run($profileName, $fresh, $gpu);
+
+ if ($gpu) {
+ $this->seedInContainer($profileName, $index > 0, $asLocalhost, true)->throw();
+
+ continue;
+ }
try {
SeedDevelopmentQemuServer::run($profileName, $index === 0, $asLocalhost);
} catch (QueryException $exception) {
- $keepOthers = $index === 0 ? '' : ' --keep-others';
- $localhostOption = $asLocalhost ? ' --as-localhost' : '';
- $container = escapeshellarg(config('development-qemu.coolify_container'));
- $result = Process::run("docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName).$keepOthers.$localhostOption);
-
- if ($result->failed()) {
+ if ($this->seedInContainer($profileName, $index > 0, $asLocalhost, false)->failed()) {
throw $exception;
}
}
}
}
+
+ private function seedInContainer(string $profileName, bool $keepOthers, bool $asLocalhost, bool $gpu): ProcessResult
+ {
+ $container = escapeshellarg(config('development-qemu.coolify_container'));
+ $command = "docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName)
+ .($keepOthers ? ' --keep-others' : '')
+ .($asLocalhost ? ' --as-localhost' : '')
+ .($gpu ? ' --gpu' : '');
+
+ return $gpu
+ ? Process::forever()->run($command, fn (string $type, string $output) => fwrite(STDOUT, $output))
+ : Process::run($command);
+ }
}
diff --git a/app/Actions/Development/StartDevelopmentQemuVm.php b/app/Actions/Development/StartDevelopmentQemuVm.php
index 5815185508..8e52a6dcc4 100644
--- a/app/Actions/Development/StartDevelopmentQemuVm.php
+++ b/app/Actions/Development/StartDevelopmentQemuVm.php
@@ -15,8 +15,9 @@ class StartDevelopmentQemuVm
/**
* Start the VM of a profile, creating it only when its libvirt domain does not exist yet.
* With $fresh, only this profile's VM is destroyed, undefined, and deleted before it is created again.
+ * With $gpu, the NVIDIA GPU of the host is passed through to the VM. An existing VM is stopped to add it.
*/
- public function handle(string $profileName, bool $fresh = false): void
+ public function handle(string $profileName, bool $fresh = false, bool $gpu = false): void
{
$this->ensureDevelopmentEnvironment();
$profile = config("development-qemu.profiles.{$profileName}");
@@ -33,9 +34,12 @@ class StartDevelopmentQemuVm
}
if (! $fresh && $this->domainExists($profile['domain'])) {
+ if ($gpu) {
+ $this->attachGpu($profile['domain']);
+ }
$this->startExistingVm($profile['domain']);
} else {
- $this->createPreparedVm($profile);
+ $this->createPreparedVm($profile, $gpu);
}
ConfigureDevelopmentQemuHost::run();
@@ -43,13 +47,13 @@ class StartDevelopmentQemuVm
}
/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */
- private function createPreparedVm(array $profile): void
+ private function createPreparedVm(array $profile, bool $gpu): void
{
$preparedImage = $this->preparedImage($profile['template']);
if (! File::exists($preparedImage)) {
$this->createVm($profile, false);
- $this->waitForPreparation($profile['domain']);
+ $this->waitForShutOff($profile['domain'], 900);
$this->runOrFail('virsh undefine '.escapeshellarg($profile['domain']));
$this->runOrFail('mv '.escapeshellarg($this->vmDisk($profile['domain'])).' '.escapeshellarg($preparedImage));
if (File::exists($preparedImage)) {
@@ -57,7 +61,7 @@ class StartDevelopmentQemuVm
}
}
- $this->createVm($profile, true);
+ $this->createVm($profile, true, $gpu);
}
private function domainExists(string $domain): bool
@@ -79,20 +83,84 @@ class StartDevelopmentQemuVm
return;
}
- $this->applyMemory($domain);
+ $hasGpu = $this->hasGpu($domain);
+ $this->applyMemory($domain, $hasGpu);
+
+ if ($hasGpu) {
+ $this->releaseHostGpu();
+ }
+
$this->runOrFail('virsh start '.escapeshellarg($domain));
}
/**
* Sets the configured memory on a stopped VM, so VMs created with a larger size shrink on their next start.
*/
- private function applyMemory(string $domain): void
+ private function applyMemory(string $domain, bool $hasGpu): void
{
- $memory = (int) config('development-qemu.memory').'M';
+ $memory = (int) config($hasGpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory').'M';
$this->runOrFail(sprintf('virsh setmaxmem %s %s --config', escapeshellarg($domain), $memory));
$this->runOrFail(sprintf('virsh setmem %s %s --config', escapeshellarg($domain), $memory));
}
+ private function hasGpu(string $domain): bool
+ {
+ return str_contains(Process::run('virsh dumpxml --inactive '.escapeshellarg($domain))->output(), '<hostdev ');
+ }
+
+ /**
+ * Adds the GPU to the persistent definition of an existing VM. A running VM is shut down first.
+ */
+ private function attachGpu(string $domain): void
+ {
+ if ($this->hasGpu($domain)) {
+ return;
+ }
+
+ if (trim(Process::run('virsh domstate '.escapeshellarg($domain))->output()) !== 'shut off') {
+ $this->runOrFail('virsh shutdown '.escapeshellarg($domain));
+ $this->waitForShutOff($domain, 120);
+ }
+
+ foreach ($this->hostGpuDevices() as $device) {
+ $this->runOrFail(sprintf('virt-xml %s --add-device --hostdev %s', escapeshellarg($domain), escapeshellarg($device)));
+ }
+ }
+
+ /**
+ * Returns the libvirt node device names of every PCI function of the first NVIDIA display controller,
+ * for example pci_0000_01_00_0 (GPU) and pci_0000_01_00_1 (its audio device). All functions must move together.
+ *
+ * @return array<int, string>
+ */
+ private function hostGpuDevices(): array
+ {
+ $devices = collect(explode("\n", trim(Process::run('lspci -Dnmm -d 10de:')->output())))
+ ->filter()
+ ->map(fn (string $line): array => str_getcsv($line, ' '));
+ $gpu = $devices->first(fn (array $device): bool => in_array($device[1] ?? null, ['0300', '0302'], true));
+
+ if ($gpu === null) {
+ throw new RuntimeException('No NVIDIA GPU found on this host.');
+ }
+
+ $slot = substr($gpu[0], 0, strrpos($gpu[0], '.'));
+
+ return $devices
+ ->filter(fn (array $device): bool => str_starts_with($device[0], "{$slot}."))
+ ->map(fn (array $device): string => 'pci_'.str_replace([':', '.'], '_', $device[0]))
+ ->values()
+ ->all();
+ }
+
+ /**
+ * nvidia-persistenced keeps the GPU open, which blocks libvirt from moving it to vfio-pci.
+ */
+ private function releaseHostGpu(): void
+ {
+ Process::run('systemctl stop nvidia-persistenced');
+ }
+
private function destroyVm(string $domain): void
{
Process::run('virsh destroy '.escapeshellarg($domain));
@@ -101,7 +169,7 @@ class StartDevelopmentQemuVm
}
/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */
- private function createVm(array $profile, bool $prepared): void
+ private function createVm(array $profile, bool $prepared, bool $gpu = false): void
{
$directory = config('development-qemu.storage_path');
File::ensureDirectoryExists($directory);
@@ -155,11 +223,17 @@ class StartDevelopmentQemuVm
$seedDisk = $prepared ? '' : ' --disk path='.escapeshellarg($seedImage).',format=raw,bus=virtio,readonly=on';
// Instance networks are isolated, so VMs of different instances may share the profile MAC.
$macCheck = config('development-qemu.instance') === null ? '' : ' --check mac_in_use=off';
+ $hostDevices = '';
+
+ if ($gpu) {
+ $hostDevices = collect($this->hostGpuDevices())->map(fn (string $device): string => ' --hostdev '.escapeshellarg($device))->implode('');
+ $this->releaseHostGpu();
+ }
$this->runOrFail(sprintf(
- 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck,
+ 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck.$hostDevices,
escapeshellarg($profile['domain']),
- config('development-qemu.memory'),
+ config($gpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory'),
config('development-qemu.vcpus'),
escapeshellarg($profile['os_variant']),
escapeshellarg($disk),
@@ -182,10 +256,10 @@ class StartDevelopmentQemuVm
return config('development-qemu.storage_path')."/{$template}-prepared.qcow2";
}
- private function waitForPreparation(string $domain): void
+ private function waitForShutOff(string $domain, int $timeoutSeconds): void
{
$check = 'while [ "$(virsh domstate '.escapeshellarg($domain).')" != "shut off" ]; do sleep 2; done';
- $this->runOrFail('timeout 900 bash -c '.escapeshellarg($check));
+ $this->runOrFail("timeout {$timeoutSeconds} bash -c ".escapeshellarg($check));
}
private function moveLegacyFiles(string $directory): void
diff --git a/app/Console/Commands/ManageDevelopmentQemuVmCommand.php b/app/Console/Commands/ManageDevelopmentQemuVmCommand.php
index 1dc235d0d0..bf27d1747e 100644
--- a/app/Console/Commands/ManageDevelopmentQemuVmCommand.php
+++ b/app/Console/Commands/ManageDevelopmentQemuVmCommand.php
@@ -9,7 +9,7 @@ use function Laravel\Prompts\multiselect;
class ManageDevelopmentQemuVmCommand extends Command
{
- protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them}';
+ protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them} {--gpu : Pass the NVIDIA GPU of the host through to the VM and install its driver}';
protected $description = 'Start (or create) selected development QEMU VMs and seed their Coolify servers';
@@ -34,7 +34,17 @@ class ManageDevelopmentQemuVmCommand extends Command
return self::FAILURE;
}
- ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'));
+ if ($this->option('gpu')) {
+ $profile = $profiles[$profileNames[0]] ?? null;
+
+ if (count($profileNames) !== 1 || $profile === null || $profile['user'] !== 'root' || ! str_starts_with($profile['os_variant'], 'ubuntu')) {
+ $this->error('GPU passthrough needs exactly one Ubuntu root profile (ubuntu-root).');
+
+ return self::FAILURE;
+ }
+ }
+
+ ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'), (bool) $this->option('gpu'));
foreach ($profileNames as $profileName) {
$profile = $profiles[$profileName];
diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php
index 26fc463ff7..aee9925163 100644
--- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php
+++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php
@@ -2,6 +2,7 @@
namespace App\Console\Commands;
+use App\Actions\Development\InstallDevelopmentGpuDriver;
use App\Actions\Development\SeedDevelopmentQemuServer;
use App\Actions\Development\StartDevelopmentInstanceDatabase;
use Illuminate\Console\Command;
@@ -9,7 +10,7 @@ use Throwable;
class SeedDevelopmentQemuServerCommand extends Command
{
- protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port}';
+ protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port} {--gpu : Install the NVIDIA driver and container toolkit in the VM}';
protected $description = 'Seed one development QEMU server in the Coolify database';
@@ -33,6 +34,12 @@ class SeedDevelopmentQemuServerCommand extends Command
}
}
+ if ($this->option('gpu')) {
+ $this->info('Installing the NVIDIA driver and container toolkit. This can take some minutes.');
+ InstallDevelopmentGpuDriver::run($server);
+ $this->info('The GPU is ready.');
+ }
+
return self::SUCCESS;
}
}
diff --git a/config/development-qemu.php b/config/development-qemu.php
index 983cd587aa..c9b30d049a 100644
--- a/config/development-qemu.php
+++ b/config/development-qemu.php
@@ -147,6 +147,8 @@ return [
'dns' => '1.1.1.1',
// MiB per VM. Many dev instances share one host, so keep this small; existing VMs get it on their next start.
'memory' => (int) env('DEVELOPMENT_QEMU_MEMORY', 1024),
+ // MiB for a VM with a passed-through GPU (dev:qemu --gpu). The NVIDIA driver needs more than the default.
+ 'gpu_memory' => (int) env('DEVELOPMENT_QEMU_GPU_MEMORY', 4096),
'vcpus' => 2,
'disk_size' => '20G',
'libvirt_network' => $developmentQemuInstance === null ? 'default' : "coolify-dev-{$developmentQemuSlot}",
diff --git a/scripts/dev b/scripts/dev
index 3102f0b9d7..952190b699 100755
--- a/scripts/dev
+++ b/scripts/dev
@@ -23,6 +23,7 @@
# redis +4, vite +5, mailpit +6/+7, minio +8/+9.
#
# Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE=<profile>
+# COOLIFY_DEV_GPU=1 passes the host NVIDIA GPU through to the KVM VM (ubuntu-root only)
#
# Side by side: COOLIFY_DEV_INSTANCE=<name> COOLIFY_DEV_SIDE_BY_SIDE=1 starts a second instance of
# this checkout without stopping the other ones (for example a transfer target, see
@@ -402,6 +403,8 @@ start_backend() {
local host="${DEV_BIND_ADDRESS:-127.0.0.1}"
[[ "$host" == 0.0.0.0 ]] && host=127.0.0.1
echo "Using KVM profile: $profile"
+ local gpu=()
+ [[ "${COOLIFY_DEV_GPU:-}" == 1 ]] && gpu=(--gpu)
# Use this instance database and encryption key, without a stale environment or config cache.
# Send logs to stderr to avoid root-owned files in storage/logs.
APP_ENV=local APP_CONFIG_CACHE="$ROOT/bootstrap/cache/dev-qemu-$$.php" \
@@ -412,7 +415,7 @@ start_backend() {
DEVELOPMENT_QEMU_SLOT="$(env_value "$instance" DEVELOPMENT_QEMU_SLOT)" \
DEVELOPMENT_QEMU_DOCKER_NETWORK="$(project_name "$instance")" \
DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$(project_name "$instance")" \
- "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost
+ "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost "${gpu[@]}"
elif [[ "$backend" == "lima" ]]; then
echo "Using Lima profile: $profile"
start_lima_vm
diff --git a/tests/Feature/DevScriptTest.php b/tests/Feature/DevScriptTest.php
index 232bde15de..79c5821c6d 100644
--- a/tests/Feature/DevScriptTest.php
+++ b/tests/Feature/DevScriptTest.php
@@ -269,6 +269,26 @@ BASH);
->and(devScriptLog())->toContain('php artisan dev:qemu ubuntu-root --as-localhost ENV=local CACHE=');
});
+it('passes the gpu option to the host qemu command when COOLIFY_DEV_GPU is 1', function () {
+ if (! function_exists('posix_geteuid') || posix_geteuid() !== 0 || ! file_exists('/dev/kvm') || filetype('/dev/kvm') !== 'char' || ! is_readable('/dev/kvm') || ! is_writable('/dev/kvm')) {
+ $this->markTestSkipped('KVM and root access are required for this launcher branch.');
+ }
+
+ file_put_contents($this->devRoot.'/bin/php', <<<'BASH'
+#!/usr/bin/env bash
+printf 'php %s|\n' "$*" >> "$DEV_TEST_LOG"
+BASH);
+
+ $withGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '1']);
+ $withoutGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '0']);
+
+ expect($withGpu->isSuccessful())->toBeTrue($withGpu->getErrorOutput())
+ ->and($withoutGpu->isSuccessful())->toBeTrue($withoutGpu->getErrorOutput())
+ ->and(devScriptLog())
+ ->toContain('php artisan dev:qemu ubuntu-root --as-localhost --gpu|')
+ ->toContain('php artisan dev:qemu ubuntu-root --as-localhost|');
+});
+
it('starts a lima vm as localhost on macos and seeds its forwarded ssh port', function () {
($this->fakeLima)();
runDevScript('main', ['start']);
diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.php
index 60c273a36a..3512a20fd2 100644
--- a/tests/Feature/DevelopmentQemuVmTest.php
+++ b/tests/Feature/DevelopmentQemuVmTest.php
@@ -690,3 +690,115 @@ function useDevelopmentQemuInstance(string $instance, int $slot, string $storage
'storage_path' => $storagePath,
]]);
}
+
+function fakeQemuHostWithGpu(string $domainState = 'running', string $domainXml = '<domain></domain>'): void
+{
+ Process::fake([
+ '* net-dumpxml *' => Process::result(output: '<network></network>'),
+ '* network inspect *' => Process::result(output: "172.18.0.0/16\n"),
+ '* dominfo *' => Process::result(output: 'exists'),
+ '* domstate *' => Process::result(output: "{$domainState}\n"),
+ '* dumpxml *' => Process::result(output: $domainXml),
+ 'lspci *' => Process::result(output: implode("\n", [
+ '0000:01:00.0 "0300" "10de" "2584" -ra1 -p00 "1043" "8976"',
+ '0000:01:00.1 "0403" "10de" "2291" -ra1 -p00 "1043" "8976"',
+ ])."\n"),
+ '*' => Process::result(),
+ ]);
+}
+
+it('creates a vm with every function of the host nvidia gpu and more memory', function () {
+ $storagePath = sys_get_temp_dir().'/coolify-qemu-gpu-create-test-'.uniqid();
+ config(['development-qemu.storage_path' => $storagePath, 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]);
+ File::ensureDirectoryExists($storagePath);
+ File::put("{$storagePath}/coolify-dev-ubuntu-root-prepared.qcow2", 'prepared image');
+ fakeQemuHostWithGpu();
+
+ StartDevelopmentQemuVm::run('ubuntu-root', true, true);
+
+ Process::assertRan(fn ($process) => $process->command === 'systemctl stop nvidia-persistenced');
+ Process::assertRan(fn ($process) => str_starts_with($process->command, 'virt-install ')
+ && str_contains($process->command, '--memory 4096 ')
+ && str_contains($process->command, "--hostdev 'pci_0000_01_00_0' --hostdev 'pci_0000_01_00_1'"));
+});
+
+it('stops an existing vm to attach the host gpu and starts it with the gpu memory', function () {
+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-attach-test-'.uniqid(), 'development-qemu.gpu_memory' => 4096]);
+ $hostdevAttached = false;
+ $ran = [];
+ Process::fake(function ($process) use (&$hostdevAttached, &$ran) {
+ $ran[] = $process->command;
+ $hostdevAttached = $hostdevAttached || str_starts_with($process->command, 'virt-xml ');
+
+ return match (true) {
+ str_contains($process->command, ' net-dumpxml ') => Process::result(output: '<network></network>'),
+ str_contains($process->command, ' network inspect ') => Process::result(output: "172.18.0.0/16\n"),
+ str_contains($process->command, ' dominfo ') => Process::result(output: 'exists'),
+ str_contains($process->command, ' domstate ') => Process::result(output: $hostdevAttached ? "shut off\n" : "running\n"),
+ str_contains($process->command, ' dumpxml ') => Process::result(output: $hostdevAttached ? "<domain><hostdev mode='subsystem'/></domain>" : '<domain></domain>'),
+ str_starts_with($process->command, 'lspci ') => Process::result(output: "0000:01:00.0 \"0300\" \"10de\" \"2584\"\n0000:01:00.1 \"0403\" \"10de\" \"2291\"\n"),
+ default => Process::result(),
+ };
+ });
+
+ StartDevelopmentQemuVm::run('ubuntu-root', false, true);
+
+ $commands = collect($ran);
+ $shutdown = $commands->search("virsh shutdown 'coolify-dev-ubuntu-root'");
+ $attach = $commands->search("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_0'");
+ expect($shutdown)->not->toBeFalse()
+ ->and($attach)->toBeGreaterThan($shutdown)
+ ->and($commands)->toContain("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_1'")
+ ->and($commands->search("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config"))->toBeGreaterThan($attach)
+ ->and($commands->search("virsh start 'coolify-dev-ubuntu-root'"))->toBeGreaterThan($commands->search('systemctl stop nvidia-persistenced'));
+ Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-install '));
+});
+
+it('keeps the gpu memory of a gpu vm that starts without the gpu option', function () {
+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-restart-test-'.uniqid(), 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]);
+ fakeQemuHostWithGpu('shut off', "<domain><hostdev mode='subsystem'/></domain>");
+
+ StartDevelopmentQemuVm::run('ubuntu-root');
+
+ Process::assertRan("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config");
+ Process::assertRan('systemctl stop nvidia-persistenced');
+ Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml '));
+ Process::assertNotRan(fn ($process) => str_contains($process->command, '1024M'));
+});
+
+it('fails before it changes the vm when the host has no nvidia gpu', function () {
+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-no-gpu-test-'.uniqid()]);
+ Process::fake([
+ '* net-dumpxml *' => Process::result(output: '<network></network>'),
+ '* network inspect *' => Process::result(output: "172.18.0.0/16\n"),
+ '* dominfo *' => Process::result(output: 'exists'),
+ '* domstate *' => Process::result(output: "shut off\n"),
+ '*' => Process::result(),
+ ]);
+
+ expect(fn () => StartDevelopmentQemuVm::run('ubuntu-root', false, true))
+ ->toThrow(RuntimeException::class, 'No NVIDIA GPU found');
+ Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml ') || str_contains($process->command, 'virsh start'));
+});
+
+it('allows gpu passthrough only for one ubuntu root profile', function (array $profiles) {
+ Process::fake();
+
+ expect(Artisan::call('dev:qemu', ['profiles' => $profiles, '--gpu' => true]))->toBe(Command::FAILURE);
+
+ Process::assertNothingRan();
+})->with([
+ 'non-root' => [['ubuntu-non-root']],
+ 'other distribution' => [['debian-root']],
+ 'two profiles' => [['ubuntu-root', 'debian-root']],
+]);
+
+it('seeds a gpu vm through the container so it installs the driver', function () {
+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-seed-test-'.uniqid()]);
+ SeedDevelopmentQemuServer::mock()->shouldNotReceive('handle');
+ fakeQemuHostWithGpu('shut off', "<domain><hostdev mode='subsystem'/></domain>");
+
+ ManageDevelopmentQemuVm::run('ubuntu-root', true, false, true);
+
+ Process::assertRan("docker exec 'coolify' php artisan dev:qemu:seed 'ubuntu-root' --as-localhost --gpu");
+});