Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
CUDA: filter devices on secondary discovery (#13317)
Daniel Hiltgen3f3083673496
diff --git a/discover/runner.go b/discover/runner.goindex 44737aa22..c963de6f8 100644--- a/discover/runner.go+++ b/discover/runner.go@@ -147,7 +147,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.wg.Add(1)go func(i int) {defer wg.Done()- extraEnvs := ml.GetVisibleDevicesEnv(devices[i : i+1])+ extraEnvs := ml.GetVisibleDevicesEnv(devices[i:i+1], true)devices[i].AddInitValidation(extraEnvs)if len(bootstrapDevices(ctx2ndPass, devices[i].LibraryPath, extraEnvs)) == 0 {slog.Debug("filtering device which didn't fully initialize",@@ -333,7 +333,8 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.defer cancel()// Apply any dev filters to avoid re-discovering unsupported devices, and get IDs correct- devFilter := ml.GetVisibleDevicesEnv(devices)+ // We avoid CUDA filters here to keep ROCm from failing to discover GPUs in a mixed environment+ devFilter := ml.GetVisibleDevicesEnv(devices, false)for dir := range libDirs {updatedDevices := bootstrapDevices(ctx, []string{ml.LibOllamaPath, dir}, devFilter)diff --git a/llm/server.go b/llm/server.goindex fa4e438d3..e9d0a030f 100644--- a/llm/server.go+++ b/llm/server.go@@ -227,7 +227,7 @@ func NewLlamaServer(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, modelPath stmodelPath,gpuLibs,status,- ml.GetVisibleDevicesEnv(gpus),+ ml.GetVisibleDevicesEnv(gpus, false),)s := llmServer{diff --git a/ml/device.go b/ml/device.goindex 7d86dfddb..f892b512d 100644--- a/ml/device.go+++ b/ml/device.go@@ -494,13 +494,14 @@ func FlashAttentionSupported(l []DeviceInfo) bool {// Given the list of GPUs this instantiation is targeted for,// figure out the visible devices environment variables-func GetVisibleDevicesEnv(l []DeviceInfo) map[string]string {+// Set mustFilter true to enable filtering of CUDA devices+func GetVisibleDevicesEnv(l []DeviceInfo, mustFilter bool) map[string]string {if len(l) == 0 {return nil}env := map[string]string{}for _, d := range l {- d.updateVisibleDevicesEnv(env)+ d.updateVisibleDevicesEnv(env, mustFilter)}return env}@@ -532,7 +533,7 @@ func (d DeviceInfo) PreferredLibrary(other DeviceInfo) bool {return false}-func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) {+func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string, mustFilter bool) {var envVar stringswitch d.Library {case "ROCm":@@ -541,8 +542,15 @@ func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) {if runtime.GOOS != "linux" {envVar = "HIP_VISIBLE_DEVICES"}+ case "CUDA":+ if !mustFilter {+ // By default we try to avoid filtering CUDA devices because ROCm also+ // looks at the CUDA env var, and gets confused in mixed vendor environments.+ return+ }+ envVar = "CUDA_VISIBLE_DEVICES"default:- // CUDA and Vulkan are not filtered via env var, but via scheduling decisions+ // Vulkan is not filtered via env var, but via scheduling decisionsreturn}v, existing := env[envVar]