Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Always filter devices (#12108)
Daniel Hiltgenead4a9a1d073
diff --git a/discover/amd_linux.go b/discover/amd_linux.goindex ebffbdf66..0f2aa0673 100644--- a/discover/amd_linux.go+++ b/discover/amd_linux.go@@ -277,6 +277,7 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {FreeMemory: (totalMemory - usedMemory),},ID: ID,+ filterID: gpuOrdinalID,Name: name,Compute: fmt.Sprintf("gfx%d%x%x", major, minor, patch),MinimumMemory: rocmMinimumMemory,@@ -394,7 +395,7 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {// Check for env var workaroundsif name == "1002:687f" { // Vega RX 56- gpuInfo.EnvWorkarounds = append(gpuInfo.EnvWorkarounds, [2]string{"HSA_ENABLE_SDMA", "0"})+ gpuInfo.EnvWorkarounds = append(gpuInfo.EnvWorkarounds, "HSA_ENABLE_SDMA=0")}// The GPU has passed all the verification steps and is supported@@ -523,19 +524,26 @@ func verifyKFDDriverAccess() error {return nil}-func rocmGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {+func rocmGetVisibleDevicesEnv(gpuInfo []GpuInfo) string {ids := []string{}for _, info := range gpuInfo {if info.Library != "rocm" {- // TODO shouldn't happen if things are wired correctly...- slog.Debug("rocmGetVisibleDevicesEnv skipping over non-rocm device", "library", info.Library)continue}- ids = append(ids, info.ID)+ // If the devices requires a numeric ID, for filtering purposes, we use the unfiltered ID number+ if _, err := strconv.Atoi(info.ID); err == nil {+ ids = append(ids, fmt.Sprintf("%d", info.filterID))+ } else {+ ids = append(ids, info.ID)+ }}+ if len(ids) == 0 {+ return ""+ }+// There are 3 potential env vars to use to select GPUs.// ROCR_VISIBLE_DEVICES supports UUID or numeric so is our preferred on linux// GPU_DEVICE_ORDINAL supports numeric IDs only// HIP_VISIBLE_DEVICES supports numeric IDs only- return "ROCR_VISIBLE_DEVICES", strings.Join(ids, ",")+ return "ROCR_VISIBLE_DEVICES=" + strings.Join(ids, ",")}diff --git a/discover/amd_windows.go b/discover/amd_windows.goindex 0659d12f8..08608ad1c 100644--- a/discover/amd_windows.go+++ b/discover/amd_windows.go@@ -111,6 +111,7 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {UnreliableFreeMemory: true,ID: strconv.Itoa(i), // TODO this is probably wrong if we specify visible devices+ filterID: i,DependencyPath: []string{libDir},MinimumMemory: rocmMinimumMemory,Name: name,@@ -200,19 +201,26 @@ func (gpus RocmGPUInfoList) RefreshFreeMemory() error {return nil}-func rocmGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {+func rocmGetVisibleDevicesEnv(gpuInfo []GpuInfo) string {ids := []string{}for _, info := range gpuInfo {if info.Library != "rocm" {- // TODO shouldn't happen if things are wired correctly...- slog.Debug("rocmGetVisibleDevicesEnv skipping over non-rocm device", "library", info.Library)continue}- ids = append(ids, info.ID)+ // If the devices requires a numeric ID, for filtering purposes, we use the unfiltered ID number+ if _, err := strconv.Atoi(info.ID); err == nil {+ ids = append(ids, fmt.Sprintf("%d", info.filterID))+ } else {+ ids = append(ids, info.ID)+ }}+ if len(ids) == 0 {+ return ""+ }+// There are 3 potential env vars to use to select GPUs.// ROCR_VISIBLE_DEVICES supports UUID or numeric but does not work on Windows// HIP_VISIBLE_DEVICES supports numeric IDs only// GPU_DEVICE_ORDINAL supports numeric IDs only- return "HIP_VISIBLE_DEVICES", strings.Join(ids, ",")+ return "HIP_VISIBLE_DEVICES=" + strings.Join(ids, ",")}diff --git a/discover/cuda_common.go b/discover/cuda_common.goindex 3c7cb6698..b539f6b32 100644--- a/discover/cuda_common.go+++ b/discover/cuda_common.go@@ -16,19 +16,6 @@ import (// Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.var CudaTegra string = os.Getenv("JETSON_JETPACK")-func cudaGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {- ids := []string{}- for _, info := range gpuInfo {- if info.Library != "cuda" {- // TODO shouldn't happen if things are wired correctly...- slog.Debug("cudaGetVisibleDevicesEnv skipping over non-cuda device", "library", info.Library)- continue- }- ids = append(ids, info.ID)- }- return "CUDA_VISIBLE_DEVICES", strings.Join(ids, ",")-}-func cudaVariant(gpuInfo CudaGPUInfo) string {if runtime.GOARCH == "arm64" && runtime.GOOS == "linux" {if CudaTegra != "" {diff --git a/discover/gpu.go b/discover/gpu.goindex f6e3c9cb1..b09626118 100644--- a/discover/gpu.go+++ b/discover/gpu.go@@ -371,6 +371,15 @@ func GetGPUInfo() GpuInfoList {}rocmGPUs, err = AMDGetGPUInfo()++ // The ID field is used in context of the filtered set of GPUS+ // so we have to replace any of these numeric IDs with their+ // placement in this set of GPUs+ for i := range rocmGPUs {+ if _, err := strconv.Atoi(rocmGPUs[i].ID); err == nil {+ rocmGPUs[i].ID = strconv.Itoa(i)+ }+ }if err != nil {bootstrapErrors = append(bootstrapErrors, err)}@@ -680,23 +689,16 @@ func getVerboseState() C.uint16_t {// Given the list of GPUs this instantiation is targeted for,// figure out the visible devices environment variable-//-// If different libraries are detected, the first one is what we use-func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {+func (l GpuInfoList) GetVisibleDevicesEnv() []string {if len(l) == 0 {- return "", ""+ return nil}- switch l[0].Library {- case "cuda":- return cudaGetVisibleDevicesEnv(l)- case "rocm":- return rocmGetVisibleDevicesEnv(l)- case "oneapi":- return oneapiGetVisibleDevicesEnv(l)- default:- slog.Debug("no filter required for library " + l[0].Library)- return "", ""+ vd := []string{}+ // Only filter the AMD GPUs at this level, let all NVIDIA devices through+ if tmp := rocmGetVisibleDevicesEnv(l); tmp != "" {+ vd = append(vd, tmp)}+ return vd}func GetSystemInfo() SystemInfo {diff --git a/discover/gpu_darwin.go b/discover/gpu_darwin.goindex dd5bf6e27..29b44ff50 100644--- a/discover/gpu_darwin.go+++ b/discover/gpu_darwin.go@@ -62,9 +62,9 @@ func GetCPUMem() (memInfo, error) {}, nil}-func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {+func (l GpuInfoList) GetVisibleDevicesEnv() []string {// No-op on darwin- return "", ""+ return nil}func GetSystemInfo() SystemInfo {diff --git a/discover/gpu_oneapi.go b/discover/gpu_oneapi.godeleted file mode 100644index 77941f5b3..000000000--- a/discover/gpu_oneapi.go+++ /dev/null@@ -1,21 +0,0 @@-//go:build linux || windows--package discover--import (- "log/slog"- "strings"-)--func oneapiGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {- ids := []string{}- for _, info := range gpuInfo {- if info.Library != "oneapi" {- // TODO shouldn't happen if things are wired correctly...- slog.Debug("oneapiGetVisibleDevicesEnv skipping over non-sycl device", "library", info.Library)- continue- }- ids = append(ids, info.ID)- }- return "ONEAPI_DEVICE_SELECTOR", "level_zero:" + strings.Join(ids, ",")-}diff --git a/discover/types.go b/discover/types.goindex 13a030fd5..1027aaac2 100644--- a/discover/types.go+++ b/discover/types.go@@ -27,8 +27,8 @@ type GpuInfo struct { // TODO better name maybe "InferenceProcessor"?// Any extra PATH/LD_LIBRARY_PATH dependencies required for the Library to operate properlyDependencyPath []string `json:"lib_path,omitempty"`- // Extra environment variables specific to the GPU as list of [key,value]- EnvWorkarounds [][2]string `json:"envs,omitempty"`+ // Extra environment variables specific to the GPU as list of [key=value]+ EnvWorkarounds []string `json:"envs,omitempty"`// Set to true if we can NOT reliably discover FreeMemory. A value of true indicates// the FreeMemory is best effort, and may over or under report actual memory usage@@ -36,9 +36,10 @@ type GpuInfo struct { // TODO better name maybe "InferenceProcessor"?UnreliableFreeMemory bool// GPU information- ID string `json:"gpu_id"` // string to use for selection of this specific GPU- Name string `json:"name"` // user friendly name if available- Compute string `json:"compute"` // Compute Capability or gfx+ ID string `json:"gpu_id"` // string to use for selection of this specific GPU+ filterID int //nolint:unused,nolintlint // AMD Workaround: The numeric ID of the device used to filter out other devices+ Name string `json:"name"` // user friendly name if available+ Compute string `json:"compute"` // Compute Capability or gfx// Driver Information - TODO no need to put this on each GPUDriverMajor int `json:"driver_major,omitempty"`diff --git a/llm/server.go b/llm/server.goindex 30cf5c360..b9ffdc6c1 100644--- a/llm/server.go+++ b/llm/server.go@@ -360,23 +360,28 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, as.cmd.Env = append(s.cmd.Env, "OLLAMA_LIBRARY_PATH="+strings.Join(ggmlPaths, string(filepath.ListSeparator)))- envWorkarounds := [][2]string{}+ envWorkarounds := []string{}for _, gpu := range gpus {envWorkarounds = append(envWorkarounds, gpu.EnvWorkarounds...)}+ // Always filter down the set of GPUs in case there are any unsupported devices that might crash+ envWorkarounds = append(envWorkarounds, gpus.GetVisibleDevicesEnv()...)pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))// Update or add the path variable with our adjusted versionpathNeeded := true+ envWorkaroundDone := make([]bool, len(envWorkarounds))for i := range s.cmd.Env {cmp := strings.SplitN(s.cmd.Env[i], "=", 2)if strings.EqualFold(cmp[0], pathEnv) {s.cmd.Env[i] = pathEnv + "=" + pathEnvValpathNeeded = false} else if len(envWorkarounds) != 0 {- for _, kv := range envWorkarounds {- if strings.EqualFold(cmp[0], kv[0]) {- s.cmd.Env[i] = kv[0] + "=" + kv[1]+ for j, kv := range envWorkarounds {+ tmp := strings.SplitN(kv, "=", 2)+ if strings.EqualFold(cmp[0], tmp[0]) {+ s.cmd.Env[i] = kv+ envWorkaroundDone[j] = true}}}@@ -384,6 +389,11 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, aif pathNeeded {s.cmd.Env = append(s.cmd.Env, pathEnv+"="+pathEnvVal)}+ for i, done := range envWorkaroundDone {+ if !done {+ s.cmd.Env = append(s.cmd.Env, envWorkarounds[i])+ }+ }slog.Info("starting runner", "cmd", s.cmd)slog.Debug("subprocess", "", filteredEnv(s.cmd.Env))