An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
feat(dev): pass the host NVIDIA GPU through to the KVM dev VM
Andras Bacsai0eb78dcf9b34
diff --git a/AGENTS.md b/AGENTS.mdindex 661a920c56..bfa80f40cf 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -29,7 +29,7 @@ Docker Compose-based dev setup with services: coolify (app, which also runs Revedocker compose -f docker-compose.yml -f docker-compose.dev.yml up -d```-The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.+The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. To pass the host NVIDIA GPU through to the KVM localhost VM (`ubuntu-root` only), start with `COOLIFY_DEV_GPU=1 ./scripts/dev start` (or `php artisan dev:qemu ubuntu-root --as-localhost --gpu`). This attaches every PCI function of the GPU, gives the VM 4 GiB, stops `nvidia-persistenced` on the host, and installs the NVIDIA driver and container toolkit in the VM. The host cannot use the GPU while the VM runs. Remove it with `--fresh` without `--gpu`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.## Testing the Self-Hosted Upgrade Processdiff --git a/app/Actions/Development/InstallDevelopmentGpuDriver.php b/app/Actions/Development/InstallDevelopmentGpuDriver.phpnew file mode 100644index 0000000000..ffe62f497c--- /dev/null+++ b/app/Actions/Development/InstallDevelopmentGpuDriver.php@@ -0,0 +1,47 @@+<?php++namespace App\Actions\Development;++use App\Models\Server;+use Lorisleiva\Actions\Concerns\AsAction;++/**+ * Installs the NVIDIA driver and the NVIDIA Container Toolkit on an Ubuntu development VM with a passed-through GPU,+ * then refreshes the server details so Coolify shows the GPU. Steps that are already done are skipped.+ *+ * @see https://ubuntu.com/server/docs/how-to/graphics/install-nvidia-drivers/+ * @see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html+ */+class InstallDevelopmentGpuDriver+{+ use AsAction;++ public function handle(Server $server): void+ {+ instant_remote_process([+ 'set -e',+ 'export DEBIAN_FRONTEND=noninteractive',+ 'if ! nvidia-smi >/dev/null 2>&1; then',+ 'apt-get update -qq',+ 'apt-get install -y -qq ubuntu-drivers-common',+ 'ubuntu-drivers install --gpgpu',+ // For example 580-server-open. The utilities package has no -open suffix.+ 'flavor=$(dpkg-query -W -f=\'${Package}\n\' \'nvidia-headless-no-dkms-*\' | sed -n \'s/^nvidia-headless-no-dkms-//p\' | head -n1)',+ // ubuntu-drivers installs the modules for the newest kernel, which can differ from the running one.+ 'apt-get install -y -qq "nvidia-utils-${flavor%-open}" "linux-modules-nvidia-${flavor}-$(uname -r)"',+ 'modprobe nvidia',+ 'nvidia-smi',+ 'fi',+ 'if ! command -v nvidia-ctk >/dev/null 2>&1; then',+ 'curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --batch --yes --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg',+ 'curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed \'s#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g\' > /etc/apt/sources.list.d/nvidia-container-toolkit.list',+ 'apt-get update -qq',+ 'apt-get install -y -qq nvidia-container-toolkit',+ 'nvidia-ctk runtime configure --runtime=docker',+ 'systemctl restart docker',+ 'fi',+ ], $server, timeout: 1800, disableMultiplexing: true, maxAttempts: 1);++ $server->gatherServerMetadata();+ }+}diff --git a/app/Actions/Development/ManageDevelopmentQemuVm.php b/app/Actions/Development/ManageDevelopmentQemuVm.phpindex 9b1a1c791f..9fab0050a4 100644--- a/app/Actions/Development/ManageDevelopmentQemuVm.php+++ b/app/Actions/Development/ManageDevelopmentQemuVm.php@@ -2,6 +2,7 @@namespace App\Actions\Development;+use Illuminate\Contracts\Process\ProcessResult;use Illuminate\Database\QueryException;use Illuminate\Support\Facades\Process;use Lorisleiva\Actions\Concerns\AsAction;@@ -12,28 +13,44 @@ class ManageDevelopmentQemuVm/*** Start (or create) the selected VMs and seed their servers. Existing VMs are reused unless $fresh is set.+ * With $gpu, the host GPU is passed through and the container seeds the server, because it installs the+ * driver over Coolify's SSH connection.** @param string|array<int, string> $profileNames*/- public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false): void+ public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false, bool $gpu = false): void{$profileNames = is_array($profileNames) ? array_values(array_unique($profileNames)) : [$profileNames];foreach ($profileNames as $index => $profileName) {- StartDevelopmentQemuVm::run($profileName, $fresh);+ StartDevelopmentQemuVm::run($profileName, $fresh, $gpu);++ if ($gpu) {+ $this->seedInContainer($profileName, $index > 0, $asLocalhost, true)->throw();++ continue;+ }try {SeedDevelopmentQemuServer::run($profileName, $index === 0, $asLocalhost);} catch (QueryException $exception) {- $keepOthers = $index === 0 ? '' : ' --keep-others';- $localhostOption = $asLocalhost ? ' --as-localhost' : '';- $container = escapeshellarg(config('development-qemu.coolify_container'));- $result = Process::run("docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName).$keepOthers.$localhostOption);-- if ($result->failed()) {+ if ($this->seedInContainer($profileName, $index > 0, $asLocalhost, false)->failed()) {throw $exception;}}}}++ private function seedInContainer(string $profileName, bool $keepOthers, bool $asLocalhost, bool $gpu): ProcessResult+ {+ $container = escapeshellarg(config('development-qemu.coolify_container'));+ $command = "docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName)+ .($keepOthers ? ' --keep-others' : '')+ .($asLocalhost ? ' --as-localhost' : '')+ .($gpu ? ' --gpu' : '');++ return $gpu+ ? Process::forever()->run($command, fn (string $type, string $output) => fwrite(STDOUT, $output))+ : Process::run($command);+ }}diff --git a/app/Actions/Development/StartDevelopmentQemuVm.php b/app/Actions/Development/StartDevelopmentQemuVm.phpindex 5815185508..8e52a6dcc4 100644--- a/app/Actions/Development/StartDevelopmentQemuVm.php+++ b/app/Actions/Development/StartDevelopmentQemuVm.php@@ -15,8 +15,9 @@ class StartDevelopmentQemuVm/*** Start the VM of a profile, creating it only when its libvirt domain does not exist yet.* With $fresh, only this profile's VM is destroyed, undefined, and deleted before it is created again.+ * With $gpu, the NVIDIA GPU of the host is passed through to the VM. An existing VM is stopped to add it.*/- public function handle(string $profileName, bool $fresh = false): void+ public function handle(string $profileName, bool $fresh = false, bool $gpu = false): void{$this->ensureDevelopmentEnvironment();$profile = config("development-qemu.profiles.{$profileName}");@@ -33,9 +34,12 @@ class StartDevelopmentQemuVm}if (! $fresh && $this->domainExists($profile['domain'])) {+ if ($gpu) {+ $this->attachGpu($profile['domain']);+ }$this->startExistingVm($profile['domain']);} else {- $this->createPreparedVm($profile);+ $this->createPreparedVm($profile, $gpu);}ConfigureDevelopmentQemuHost::run();@@ -43,13 +47,13 @@ class StartDevelopmentQemuVm}/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */- private function createPreparedVm(array $profile): void+ private function createPreparedVm(array $profile, bool $gpu): void{$preparedImage = $this->preparedImage($profile['template']);if (! File::exists($preparedImage)) {$this->createVm($profile, false);- $this->waitForPreparation($profile['domain']);+ $this->waitForShutOff($profile['domain'], 900);$this->runOrFail('virsh undefine '.escapeshellarg($profile['domain']));$this->runOrFail('mv '.escapeshellarg($this->vmDisk($profile['domain'])).' '.escapeshellarg($preparedImage));if (File::exists($preparedImage)) {@@ -57,7 +61,7 @@ class StartDevelopmentQemuVm}}- $this->createVm($profile, true);+ $this->createVm($profile, true, $gpu);}private function domainExists(string $domain): bool@@ -79,20 +83,84 @@ class StartDevelopmentQemuVmreturn;}- $this->applyMemory($domain);+ $hasGpu = $this->hasGpu($domain);+ $this->applyMemory($domain, $hasGpu);++ if ($hasGpu) {+ $this->releaseHostGpu();+ }+$this->runOrFail('virsh start '.escapeshellarg($domain));}/*** Sets the configured memory on a stopped VM, so VMs created with a larger size shrink on their next start.*/- private function applyMemory(string $domain): void+ private function applyMemory(string $domain, bool $hasGpu): void{- $memory = (int) config('development-qemu.memory').'M';+ $memory = (int) config($hasGpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory').'M';$this->runOrFail(sprintf('virsh setmaxmem %s %s --config', escapeshellarg($domain), $memory));$this->runOrFail(sprintf('virsh setmem %s %s --config', escapeshellarg($domain), $memory));}+ private function hasGpu(string $domain): bool+ {+ return str_contains(Process::run('virsh dumpxml --inactive '.escapeshellarg($domain))->output(), '<hostdev ');+ }++ /**+ * Adds the GPU to the persistent definition of an existing VM. A running VM is shut down first.+ */+ private function attachGpu(string $domain): void+ {+ if ($this->hasGpu($domain)) {+ return;+ }++ if (trim(Process::run('virsh domstate '.escapeshellarg($domain))->output()) !== 'shut off') {+ $this->runOrFail('virsh shutdown '.escapeshellarg($domain));+ $this->waitForShutOff($domain, 120);+ }++ foreach ($this->hostGpuDevices() as $device) {+ $this->runOrFail(sprintf('virt-xml %s --add-device --hostdev %s', escapeshellarg($domain), escapeshellarg($device)));+ }+ }++ /**+ * Returns the libvirt node device names of every PCI function of the first NVIDIA display controller,+ * for example pci_0000_01_00_0 (GPU) and pci_0000_01_00_1 (its audio device). All functions must move together.+ *+ * @return array<int, string>+ */+ private function hostGpuDevices(): array+ {+ $devices = collect(explode("\n", trim(Process::run('lspci -Dnmm -d 10de:')->output())))+ ->filter()+ ->map(fn (string $line): array => str_getcsv($line, ' '));+ $gpu = $devices->first(fn (array $device): bool => in_array($device[1] ?? null, ['0300', '0302'], true));++ if ($gpu === null) {+ throw new RuntimeException('No NVIDIA GPU found on this host.');+ }++ $slot = substr($gpu[0], 0, strrpos($gpu[0], '.'));++ return $devices+ ->filter(fn (array $device): bool => str_starts_with($device[0], "{$slot}."))+ ->map(fn (array $device): string => 'pci_'.str_replace([':', '.'], '_', $device[0]))+ ->values()+ ->all();+ }++ /**+ * nvidia-persistenced keeps the GPU open, which blocks libvirt from moving it to vfio-pci.+ */+ private function releaseHostGpu(): void+ {+ Process::run('systemctl stop nvidia-persistenced');+ }+private function destroyVm(string $domain): void{Process::run('virsh destroy '.escapeshellarg($domain));@@ -101,7 +169,7 @@ class StartDevelopmentQemuVm}/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */- private function createVm(array $profile, bool $prepared): void+ private function createVm(array $profile, bool $prepared, bool $gpu = false): void{$directory = config('development-qemu.storage_path');File::ensureDirectoryExists($directory);@@ -155,11 +223,17 @@ class StartDevelopmentQemuVm$seedDisk = $prepared ? '' : ' --disk path='.escapeshellarg($seedImage).',format=raw,bus=virtio,readonly=on';// Instance networks are isolated, so VMs of different instances may share the profile MAC.$macCheck = config('development-qemu.instance') === null ? '' : ' --check mac_in_use=off';+ $hostDevices = '';++ if ($gpu) {+ $hostDevices = collect($this->hostGpuDevices())->map(fn (string $device): string => ' --hostdev '.escapeshellarg($device))->implode('');+ $this->releaseHostGpu();+ }$this->runOrFail(sprintf(- 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck,+ 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck.$hostDevices,escapeshellarg($profile['domain']),- config('development-qemu.memory'),+ config($gpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory'),config('development-qemu.vcpus'),escapeshellarg($profile['os_variant']),escapeshellarg($disk),@@ -182,10 +256,10 @@ class StartDevelopmentQemuVmreturn config('development-qemu.storage_path')."/{$template}-prepared.qcow2";}- private function waitForPreparation(string $domain): void+ private function waitForShutOff(string $domain, int $timeoutSeconds): void{$check = 'while [ "$(virsh domstate '.escapeshellarg($domain).')" != "shut off" ]; do sleep 2; done';- $this->runOrFail('timeout 900 bash -c '.escapeshellarg($check));+ $this->runOrFail("timeout {$timeoutSeconds} bash -c ".escapeshellarg($check));}private function moveLegacyFiles(string $directory): voiddiff --git a/app/Console/Commands/ManageDevelopmentQemuVmCommand.php b/app/Console/Commands/ManageDevelopmentQemuVmCommand.phpindex 1dc235d0d0..bf27d1747e 100644--- a/app/Console/Commands/ManageDevelopmentQemuVmCommand.php+++ b/app/Console/Commands/ManageDevelopmentQemuVmCommand.php@@ -9,7 +9,7 @@ use function Laravel\Prompts\multiselect;class ManageDevelopmentQemuVmCommand extends Command{- protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them}';+ protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them} {--gpu : Pass the NVIDIA GPU of the host through to the VM and install its driver}';protected $description = 'Start (or create) selected development QEMU VMs and seed their Coolify servers';@@ -34,7 +34,17 @@ class ManageDevelopmentQemuVmCommand extends Commandreturn self::FAILURE;}- ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'));+ if ($this->option('gpu')) {+ $profile = $profiles[$profileNames[0]] ?? null;++ if (count($profileNames) !== 1 || $profile === null || $profile['user'] !== 'root' || ! str_starts_with($profile['os_variant'], 'ubuntu')) {+ $this->error('GPU passthrough needs exactly one Ubuntu root profile (ubuntu-root).');++ return self::FAILURE;+ }+ }++ ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'), (bool) $this->option('gpu'));foreach ($profileNames as $profileName) {$profile = $profiles[$profileName];diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.phpindex 26fc463ff7..aee9925163 100644--- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php+++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php@@ -2,6 +2,7 @@namespace App\Console\Commands;+use App\Actions\Development\InstallDevelopmentGpuDriver;use App\Actions\Development\SeedDevelopmentQemuServer;use App\Actions\Development\StartDevelopmentInstanceDatabase;use Illuminate\Console\Command;@@ -9,7 +10,7 @@ use Throwable;class SeedDevelopmentQemuServerCommand extends Command{- protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port}';+ protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port} {--gpu : Install the NVIDIA driver and container toolkit in the VM}';protected $description = 'Seed one development QEMU server in the Coolify database';@@ -33,6 +34,12 @@ class SeedDevelopmentQemuServerCommand extends Command}}+ if ($this->option('gpu')) {+ $this->info('Installing the NVIDIA driver and container toolkit. This can take some minutes.');+ InstallDevelopmentGpuDriver::run($server);+ $this->info('The GPU is ready.');+ }+return self::SUCCESS;}}diff --git a/config/development-qemu.php b/config/development-qemu.phpindex 983cd587aa..c9b30d049a 100644--- a/config/development-qemu.php+++ b/config/development-qemu.php@@ -147,6 +147,8 @@ return ['dns' => '1.1.1.1',// MiB per VM. Many dev instances share one host, so keep this small; existing VMs get it on their next start.'memory' => (int) env('DEVELOPMENT_QEMU_MEMORY', 1024),+ // MiB for a VM with a passed-through GPU (dev:qemu --gpu). The NVIDIA driver needs more than the default.+ 'gpu_memory' => (int) env('DEVELOPMENT_QEMU_GPU_MEMORY', 4096),'vcpus' => 2,'disk_size' => '20G','libvirt_network' => $developmentQemuInstance === null ? 'default' : "coolify-dev-{$developmentQemuSlot}",diff --git a/scripts/dev b/scripts/devindex 3102f0b9d7..952190b699 100755--- a/scripts/dev+++ b/scripts/dev@@ -23,6 +23,7 @@# redis +4, vite +5, mailpit +6/+7, minio +8/+9.## Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE=<profile>+# COOLIFY_DEV_GPU=1 passes the host NVIDIA GPU through to the KVM VM (ubuntu-root only)## Side by side: COOLIFY_DEV_INSTANCE=<name> COOLIFY_DEV_SIDE_BY_SIDE=1 starts a second instance of# this checkout without stopping the other ones (for example a transfer target, see@@ -402,6 +403,8 @@ start_backend() {local host="${DEV_BIND_ADDRESS:-127.0.0.1}"[[ "$host" == 0.0.0.0 ]] && host=127.0.0.1echo "Using KVM profile: $profile"+ local gpu=()+ [[ "${COOLIFY_DEV_GPU:-}" == 1 ]] && gpu=(--gpu)# Use this instance database and encryption key, without a stale environment or config cache.# Send logs to stderr to avoid root-owned files in storage/logs.APP_ENV=local APP_CONFIG_CACHE="$ROOT/bootstrap/cache/dev-qemu-$$.php" \@@ -412,7 +415,7 @@ start_backend() {DEVELOPMENT_QEMU_SLOT="$(env_value "$instance" DEVELOPMENT_QEMU_SLOT)" \DEVELOPMENT_QEMU_DOCKER_NETWORK="$(project_name "$instance")" \DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$(project_name "$instance")" \- "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost+ "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost "${gpu[@]}"elif [[ "$backend" == "lima" ]]; thenecho "Using Lima profile: $profile"start_lima_vmdiff --git a/tests/Feature/DevScriptTest.php b/tests/Feature/DevScriptTest.phpindex 232bde15de..79c5821c6d 100644--- a/tests/Feature/DevScriptTest.php+++ b/tests/Feature/DevScriptTest.php@@ -269,6 +269,26 @@ BASH);->and(devScriptLog())->toContain('php artisan dev:qemu ubuntu-root --as-localhost ENV=local CACHE=');});+it('passes the gpu option to the host qemu command when COOLIFY_DEV_GPU is 1', function () {+ if (! function_exists('posix_geteuid') || posix_geteuid() !== 0 || ! file_exists('/dev/kvm') || filetype('/dev/kvm') !== 'char' || ! is_readable('/dev/kvm') || ! is_writable('/dev/kvm')) {+ $this->markTestSkipped('KVM and root access are required for this launcher branch.');+ }++ file_put_contents($this->devRoot.'/bin/php', <<<'BASH'+#!/usr/bin/env bash+printf 'php %s|\n' "$*" >> "$DEV_TEST_LOG"+BASH);++ $withGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '1']);+ $withoutGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '0']);++ expect($withGpu->isSuccessful())->toBeTrue($withGpu->getErrorOutput())+ ->and($withoutGpu->isSuccessful())->toBeTrue($withoutGpu->getErrorOutput())+ ->and(devScriptLog())+ ->toContain('php artisan dev:qemu ubuntu-root --as-localhost --gpu|')+ ->toContain('php artisan dev:qemu ubuntu-root --as-localhost|');+});+it('starts a lima vm as localhost on macos and seeds its forwarded ssh port', function () {($this->fakeLima)();runDevScript('main', ['start']);diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.phpindex 60c273a36a..3512a20fd2 100644--- a/tests/Feature/DevelopmentQemuVmTest.php+++ b/tests/Feature/DevelopmentQemuVmTest.php@@ -690,3 +690,115 @@ function useDevelopmentQemuInstance(string $instance, int $slot, string $storage'storage_path' => $storagePath,]]);}++function fakeQemuHostWithGpu(string $domainState = 'running', string $domainXml = '<domain></domain>'): void+{+ Process::fake([+ '* net-dumpxml *' => Process::result(output: '<network></network>'),+ '* network inspect *' => Process::result(output: "172.18.0.0/16\n"),+ '* dominfo *' => Process::result(output: 'exists'),+ '* domstate *' => Process::result(output: "{$domainState}\n"),+ '* dumpxml *' => Process::result(output: $domainXml),+ 'lspci *' => Process::result(output: implode("\n", [+ '0000:01:00.0 "0300" "10de" "2584" -ra1 -p00 "1043" "8976"',+ '0000:01:00.1 "0403" "10de" "2291" -ra1 -p00 "1043" "8976"',+ ])."\n"),+ '*' => Process::result(),+ ]);+}++it('creates a vm with every function of the host nvidia gpu and more memory', function () {+ $storagePath = sys_get_temp_dir().'/coolify-qemu-gpu-create-test-'.uniqid();+ config(['development-qemu.storage_path' => $storagePath, 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]);+ File::ensureDirectoryExists($storagePath);+ File::put("{$storagePath}/coolify-dev-ubuntu-root-prepared.qcow2", 'prepared image');+ fakeQemuHostWithGpu();++ StartDevelopmentQemuVm::run('ubuntu-root', true, true);++ Process::assertRan(fn ($process) => $process->command === 'systemctl stop nvidia-persistenced');+ Process::assertRan(fn ($process) => str_starts_with($process->command, 'virt-install ')+ && str_contains($process->command, '--memory 4096 ')+ && str_contains($process->command, "--hostdev 'pci_0000_01_00_0' --hostdev 'pci_0000_01_00_1'"));+});++it('stops an existing vm to attach the host gpu and starts it with the gpu memory', function () {+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-attach-test-'.uniqid(), 'development-qemu.gpu_memory' => 4096]);+ $hostdevAttached = false;+ $ran = [];+ Process::fake(function ($process) use (&$hostdevAttached, &$ran) {+ $ran[] = $process->command;+ $hostdevAttached = $hostdevAttached || str_starts_with($process->command, 'virt-xml ');++ return match (true) {+ str_contains($process->command, ' net-dumpxml ') => Process::result(output: '<network></network>'),+ str_contains($process->command, ' network inspect ') => Process::result(output: "172.18.0.0/16\n"),+ str_contains($process->command, ' dominfo ') => Process::result(output: 'exists'),+ str_contains($process->command, ' domstate ') => Process::result(output: $hostdevAttached ? "shut off\n" : "running\n"),+ str_contains($process->command, ' dumpxml ') => Process::result(output: $hostdevAttached ? "<domain><hostdev mode='subsystem'/></domain>" : '<domain></domain>'),+ str_starts_with($process->command, 'lspci ') => Process::result(output: "0000:01:00.0 \"0300\" \"10de\" \"2584\"\n0000:01:00.1 \"0403\" \"10de\" \"2291\"\n"),+ default => Process::result(),+ };+ });++ StartDevelopmentQemuVm::run('ubuntu-root', false, true);++ $commands = collect($ran);+ $shutdown = $commands->search("virsh shutdown 'coolify-dev-ubuntu-root'");+ $attach = $commands->search("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_0'");+ expect($shutdown)->not->toBeFalse()+ ->and($attach)->toBeGreaterThan($shutdown)+ ->and($commands)->toContain("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_1'")+ ->and($commands->search("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config"))->toBeGreaterThan($attach)+ ->and($commands->search("virsh start 'coolify-dev-ubuntu-root'"))->toBeGreaterThan($commands->search('systemctl stop nvidia-persistenced'));+ Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-install '));+});++it('keeps the gpu memory of a gpu vm that starts without the gpu option', function () {+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-restart-test-'.uniqid(), 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]);+ fakeQemuHostWithGpu('shut off', "<domain><hostdev mode='subsystem'/></domain>");++ StartDevelopmentQemuVm::run('ubuntu-root');++ Process::assertRan("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config");+ Process::assertRan('systemctl stop nvidia-persistenced');+ Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml '));+ Process::assertNotRan(fn ($process) => str_contains($process->command, '1024M'));+});++it('fails before it changes the vm when the host has no nvidia gpu', function () {+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-no-gpu-test-'.uniqid()]);+ Process::fake([+ '* net-dumpxml *' => Process::result(output: '<network></network>'),+ '* network inspect *' => Process::result(output: "172.18.0.0/16\n"),+ '* dominfo *' => Process::result(output: 'exists'),+ '* domstate *' => Process::result(output: "shut off\n"),+ '*' => Process::result(),+ ]);++ expect(fn () => StartDevelopmentQemuVm::run('ubuntu-root', false, true))+ ->toThrow(RuntimeException::class, 'No NVIDIA GPU found');+ Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml ') || str_contains($process->command, 'virsh start'));+});++it('allows gpu passthrough only for one ubuntu root profile', function (array $profiles) {+ Process::fake();++ expect(Artisan::call('dev:qemu', ['profiles' => $profiles, '--gpu' => true]))->toBe(Command::FAILURE);++ Process::assertNothingRan();+})->with([+ 'non-root' => [['ubuntu-non-root']],+ 'other distribution' => [['debian-root']],+ 'two profiles' => [['ubuntu-root', 'debian-root']],+]);++it('seeds a gpu vm through the container so it installs the driver', function () {+ config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-seed-test-'.uniqid()]);+ SeedDevelopmentQemuServer::mock()->shouldNotReceive('handle');+ fakeQemuHostWithGpu('shut off', "<domain><hostdev mode='subsystem'/></domain>");++ ManageDevelopmentQemuVm::run('ubuntu-root', true, false, true);++ Process::assertRan("docker exec 'coolify' php artisan dev:qemu:seed 'ubuntu-root' --as-localhost --gpu");+});