Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Rename gpu package discover (#7143)
Daniel Hiltgen05cd82ef94a5
diff --git a/gpu/amd_common.go b/discover/amd_common.gosimilarity index 99%rename from gpu/amd_common.gorename to discover/amd_common.goindex 2894ac2c4..bf9692402 100644--- a/gpu/amd_common.go+++ b/discover/amd_common.go@@ -1,6 +1,6 @@//go:build linux || windows-package gpu+package discoverimport ("errors"diff --git a/gpu/amd_hip_windows.go b/discover/amd_hip_windows.gosimilarity index 99%rename from gpu/amd_hip_windows.gorename to discover/amd_hip_windows.goindex 2cea28242..12a7af0be 100644--- a/gpu/amd_hip_windows.go+++ b/discover/amd_hip_windows.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("errors"diff --git a/gpu/amd_linux.go b/discover/amd_linux.gosimilarity index 99%rename from gpu/amd_linux.gorename to discover/amd_linux.goindex 72dfb4db3..dd8e605c3 100644--- a/gpu/amd_linux.go+++ b/discover/amd_linux.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("bufio"diff --git a/gpu/amd_windows.go b/discover/amd_windows.gosimilarity index 99%rename from gpu/amd_windows.gorename to discover/amd_windows.goindex 4da6b7cc6..a3a6e0c35 100644--- a/gpu/amd_windows.go+++ b/discover/amd_windows.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("bytes"diff --git a/gpu/cpu_common.go b/discover/cpu_common.gosimilarity index 97%rename from gpu/cpu_common.gorename to discover/cpu_common.goindex 34edcdc5a..0faac24ac 100644--- a/gpu/cpu_common.go+++ b/discover/cpu_common.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("os"diff --git a/gpu/cuda_common.go b/discover/cuda_common.gosimilarity index 99%rename from gpu/cuda_common.gorename to discover/cuda_common.goindex aceec70af..878cee8cb 100644--- a/gpu/cuda_common.go+++ b/discover/cuda_common.go@@ -1,6 +1,6 @@//go:build linux || windows-package gpu+package discoverimport ("log/slog"diff --git a/gpu/gpu.go b/discover/gpu.gosimilarity index 99%rename from gpu/gpu.gorename to discover/gpu.goindex 700c90f9e..4f2e08847 100644--- a/gpu/gpu.go+++ b/discover/gpu.go@@ -1,6 +1,6 @@//go:build linux || windows-package gpu+package discover/*#cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lmdiff --git a/gpu/gpu_darwin.go b/discover/gpu_darwin.gosimilarity index 99%rename from gpu/gpu_darwin.gorename to discover/gpu_darwin.goindex 530b98c9b..d3f0303f7 100644--- a/gpu/gpu_darwin.go+++ b/discover/gpu_darwin.go@@ -1,6 +1,6 @@//go:build darwin-package gpu+package discover/*#cgo CFLAGS: -x objective-cdiff --git a/gpu/gpu_info.h b/discover/gpu_info.hsimilarity index 100%rename from gpu/gpu_info.hrename to discover/gpu_info.hdiff --git a/gpu/gpu_info_cudart.c b/discover/gpu_info_cudart.csimilarity index 100%rename from gpu/gpu_info_cudart.crename to discover/gpu_info_cudart.cdiff --git a/gpu/gpu_info_cudart.h b/discover/gpu_info_cudart.hsimilarity index 100%rename from gpu/gpu_info_cudart.hrename to discover/gpu_info_cudart.hdiff --git a/gpu/gpu_info_darwin.h b/discover/gpu_info_darwin.hsimilarity index 100%rename from gpu/gpu_info_darwin.hrename to discover/gpu_info_darwin.hdiff --git a/gpu/gpu_info_darwin.m b/discover/gpu_info_darwin.msimilarity index 100%rename from gpu/gpu_info_darwin.mrename to discover/gpu_info_darwin.mdiff --git a/gpu/gpu_info_nvcuda.c b/discover/gpu_info_nvcuda.csimilarity index 100%rename from gpu/gpu_info_nvcuda.crename to discover/gpu_info_nvcuda.cdiff --git a/gpu/gpu_info_nvcuda.h b/discover/gpu_info_nvcuda.hsimilarity index 100%rename from gpu/gpu_info_nvcuda.hrename to discover/gpu_info_nvcuda.hdiff --git a/gpu/gpu_info_nvml.c b/discover/gpu_info_nvml.csimilarity index 100%rename from gpu/gpu_info_nvml.crename to discover/gpu_info_nvml.cdiff --git a/gpu/gpu_info_nvml.h b/discover/gpu_info_nvml.hsimilarity index 100%rename from gpu/gpu_info_nvml.hrename to discover/gpu_info_nvml.hdiff --git a/gpu/gpu_info_oneapi.c b/discover/gpu_info_oneapi.csimilarity index 100%rename from gpu/gpu_info_oneapi.crename to discover/gpu_info_oneapi.cdiff --git a/gpu/gpu_info_oneapi.h b/discover/gpu_info_oneapi.hsimilarity index 100%rename from gpu/gpu_info_oneapi.hrename to discover/gpu_info_oneapi.hdiff --git a/gpu/gpu_linux.go b/discover/gpu_linux.gosimilarity index 99%rename from gpu/gpu_linux.gorename to discover/gpu_linux.goindex b15bc2d2c..9273baeb9 100644--- a/gpu/gpu_linux.go+++ b/discover/gpu_linux.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("bufio"diff --git a/gpu/gpu_oneapi.go b/discover/gpu_oneapi.gosimilarity index 96%rename from gpu/gpu_oneapi.gorename to discover/gpu_oneapi.goindex 9864bde51..77941f5b3 100644--- a/gpu/gpu_oneapi.go+++ b/discover/gpu_oneapi.go@@ -1,6 +1,6 @@//go:build linux || windows-package gpu+package discoverimport ("log/slog"diff --git a/gpu/gpu_test.go b/discover/gpu_test.gosimilarity index 99%rename from gpu/gpu_test.gorename to discover/gpu_test.goindex 13a3f5442..0c6ef7bad 100644--- a/gpu/gpu_test.go+++ b/discover/gpu_test.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("runtime"diff --git a/gpu/gpu_windows.go b/discover/gpu_windows.gosimilarity index 99%rename from gpu/gpu_windows.gorename to discover/gpu_windows.goindex da0dce928..e7665a63c 100644--- a/gpu/gpu_windows.go+++ b/discover/gpu_windows.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("fmt"diff --git a/gpu/gpu_windows_test.go b/discover/gpu_windows_test.gosimilarity index 99%rename from gpu/gpu_windows_test.gorename to discover/gpu_windows_test.goindex 27e609cfd..c4daa7b72 100644--- a/gpu/gpu_windows_test.go+++ b/discover/gpu_windows_test.go@@ -1,4 +1,4 @@-package gpu+package discoverimport "testing"diff --git a/gpu/types.go b/discover/types.gosimilarity index 99%rename from gpu/types.gorename to discover/types.goindex 655c8733f..56b168c45 100644--- a/gpu/types.go+++ b/discover/types.go@@ -1,4 +1,4 @@-package gpu+package discoverimport ("fmt"diff --git a/llm/memory.go b/llm/memory.goindex 99db76299..d8dbf0bea 100644--- a/llm/memory.go+++ b/llm/memory.go@@ -7,13 +7,13 @@ import ("strings""github.com/ollama/ollama/api"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format"- "github.com/ollama/ollama/gpu")// This algorithm looks for a complete fit to determine if we need to unload other models-func PredictServerFit(allGpus gpu.GpuInfoList, ggml *GGML, adapters, projectors []string, opts api.Options) (bool, uint64) {+func PredictServerFit(allGpus discover.GpuInfoList, ggml *GGML, adapters, projectors []string, opts api.Options) (bool, uint64) {// Split up the GPUs by type and try themvar estimatedVRAM uint64for _, gpus := range allGpus.ByLibrary() {@@ -67,7 +67,7 @@ type MemoryEstimate struct {// Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size// The GPUs provided must all be the same Library-func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts api.Options) MemoryEstimate {+func EstimateGPULayers(gpus []discover.GpuInfo, ggml *GGML, projectors []string, opts api.Options) MemoryEstimate {// Graph size for a partial offload, applies to all GPUsvar graphPartialOffload uint64@@ -157,7 +157,7 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, optsgpuAllocations := make([]uint64, len(gpus))type gs struct {i int- g *gpu.GpuInfo+ g *discover.GpuInfo}gpusWithSpace := []gs{}for i := range gpus {diff --git a/llm/memory_test.go b/llm/memory_test.goindex ffb14286b..73e77d908 100644--- a/llm/memory_test.go+++ b/llm/memory_test.go@@ -10,7 +10,7 @@ import ("github.com/stretchr/testify/require""github.com/ollama/ollama/api"- "github.com/ollama/ollama/gpu"+ "github.com/ollama/ollama/discover")func TestEstimateGPULayers(t *testing.T) {@@ -50,7 +50,7 @@ func TestEstimateGPULayers(t *testing.T) {}// Simple CPU scenario- gpus := []gpu.GpuInfo{+ gpus := []discover.GpuInfo{{Library: "cpu",},@@ -72,7 +72,7 @@ func TestEstimateGPULayers(t *testing.T) {// Dual CUDA scenario with assymetrygpuMinimumMemory := uint64(2048)- gpus = []gpu.GpuInfo{+ gpus = []discover.GpuInfo{{Library: "cuda",MinimumMemory: gpuMinimumMemory,diff --git a/llm/server.go b/llm/server.goindex 36bcfad21..03aeeb0eb 100644--- a/llm/server.go+++ b/llm/server.go@@ -26,9 +26,9 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/build"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format"- "github.com/ollama/ollama/gpu""github.com/ollama/ollama/llama""github.com/ollama/ollama/runners")@@ -61,8 +61,8 @@ type llmServer struct {estimate MemoryEstimatetotalLayers uint64// gpuCount int- gpus gpu.GpuInfoList // Recorded just before the model loaded, free space will be incorrect- loadDuration time.Duration // Record how long it took the model to load+ gpus discover.GpuInfoList // Recorded just before the model loaded, free space will be incorrect+ loadDuration time.Duration // Record how long it took the model to loadloadProgress float32sem *semaphore.Weighted@@ -90,7 +90,7 @@ func LoadModel(model string, maxArraySize int) (*GGML, error) {// NewLlamaServer will run a server for the given GPUs// The gpu list must be a single family.-func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {+func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {var err errorvar cpuRunner stringvar estimate MemoryEstimate@@ -98,7 +98,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, prvar systemFreeMemory uint64var systemSwapFreeMemory uint64- systemInfo := gpu.GetSystemInfo()+ systemInfo := discover.GetSystemInfo()systemTotalMemory = systemInfo.System.TotalMemorysystemFreeMemory = systemInfo.System.FreeMemorysystemSwapFreeMemory = systemInfo.System.FreeSwap@@ -106,7 +106,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr// If the user wants zero GPU layers, reset the gpu list to be CPU/system ram infoif opts.NumGPU == 0 {- gpus = gpu.GetCPUInfo()+ gpus = discover.GetCPUInfo()}if len(gpus) == 1 && gpus[0].Library == "cpu" {cpuRunner = runners.ServerForCpu()@@ -122,7 +122,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, prcase gpus[0].Library != "metal" && estimate.Layers == 0:// Don't bother loading into the GPU if no layers can fitcpuRunner = runners.ServerForCpu()- gpus = gpu.GetCPUInfo()+ gpus = discover.GetCPUInfo()case opts.NumGPU < 0 && estimate.Layers > 0 && gpus[0].Library != "cpu":opts.NumGPU = estimate.Layers}@@ -281,7 +281,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr}if strings.HasPrefix(servers[i], "cpu") {- gpus = gpu.GetCPUInfo()+ gpus = discover.GetCPUInfo()}// Find an availableServers port, retry on each iteration in case the failure was a port conflict racediff --git a/runners/common.go b/runners/common.goindex 681c397b9..19014d75c 100644--- a/runners/common.go+++ b/runners/common.go@@ -18,8 +18,8 @@ import ("golang.org/x/sync/errgroup"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig"- "github.com/ollama/ollama/gpu")const (@@ -301,11 +301,11 @@ func GetAvailableServers(payloadsDir string) map[string]string {// serversForGpu returns a list of compatible servers give the provided GPU// info, ordered by performance. assumes Init() has been called// TODO - switch to metadata based mapping-func ServersForGpu(info gpu.GpuInfo) []string {+func ServersForGpu(info discover.GpuInfo) []string {// glob workDir for files that start with ollama_availableServers := GetAvailableServers(runnersDir)requested := info.Library- if info.Variant != gpu.CPUCapabilityNone.String() {+ if info.Variant != discover.CPUCapabilityNone.String() {requested += "_" + info.Variant}@@ -341,12 +341,12 @@ func ServersForGpu(info gpu.GpuInfo) []string {if !(runtime.GOOS == "darwin" && runtime.GOARCH == "arm64") {// Load up the best CPU variant if not primary requestedif info.Library != "cpu" {- variant := gpu.GetCPUCapability()+ variant := discover.GetCPUCapability()// If no variant, then we fall back to default// If we have a variant, try that if we find an exact match// Attempting to run the wrong CPU instructions will panic the// process- if variant != gpu.CPUCapabilityNone {+ if variant != discover.CPUCapabilityNone {for cmp := range availableServers {if cmp == "cpu_"+variant.String() {servers = append(servers, cmp)@@ -371,9 +371,9 @@ func ServerForCpu() string {if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {return "metal"}- variant := gpu.GetCPUCapability()+ variant := discover.GetCPUCapability()availableServers := GetAvailableServers(runnersDir)- if variant != gpu.CPUCapabilityNone {+ if variant != discover.CPUCapabilityNone {for cmp := range availableServers {if cmp == "cpu_"+variant.String() {return cmpdiff --git a/server/routes.go b/server/routes.goindex 23f9dbfd9..c2b9b2418 100644--- a/server/routes.go+++ b/server/routes.go@@ -27,8 +27,8 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/build"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig"- "github.com/ollama/ollama/gpu""github.com/ollama/ollama/llm""github.com/ollama/ollama/openai""github.com/ollama/ollama/parser"@@ -1235,7 +1235,7 @@ func Serve(ln net.Listener) error {// At startup we retrieve GPU information so we can get log messages before loading a model// This will log warnings to the log in case we have problems with detected GPUs- gpus := gpu.GetGPUInfo()+ gpus := discover.GetGPUInfo()gpus.LogDetails()err = srvr.Serve(ln)diff --git a/server/routes_generate_test.go b/server/routes_generate_test.goindex 480b96721..9cadf56a7 100644--- a/server/routes_generate_test.go+++ b/server/routes_generate_test.go@@ -15,7 +15,7 @@ import ("github.com/google/go-cmp/cmp""github.com/ollama/ollama/api"- "github.com/ollama/ollama/gpu"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/llm")@@ -41,8 +41,8 @@ func (mockRunner) Tokenize(_ context.Context, s string) (tokens []int, err errorreturn}-func newMockServer(mock *mockRunner) func(gpu.GpuInfoList, string, *llm.GGML, []string, []string, api.Options, int) (llm.LlamaServer, error) {- return func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, projectors, system []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+func newMockServer(mock *mockRunner) func(discover.GpuInfoList, string, *llm.GGML, []string, []string, api.Options, int) (llm.LlamaServer, error) {+ return func(gpus discover.GpuInfoList, model string, ggml *llm.GGML, projectors, system []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {return mock, nil}}@@ -69,10 +69,10 @@ func TestGenerateChat(t *testing.T) {unloadedCh: make(chan any, 1),loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),- getGpuFn: gpu.GetGPUInfo,- getCpuFn: gpu.GetCPUInfo,+ getGpuFn: discover.GetGPUInfo,+ getCpuFn: discover.GetCPUInfo,reschedDelay: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int) {+ loadFn: func(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList, numParallel int) {// add small delay to simulate loadingtime.Sleep(time.Millisecond)req.successCh <- &runnerRef{@@ -367,10 +367,10 @@ func TestGenerate(t *testing.T) {unloadedCh: make(chan any, 1),loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),- getGpuFn: gpu.GetGPUInfo,- getCpuFn: gpu.GetCPUInfo,+ getGpuFn: discover.GetGPUInfo,+ getCpuFn: discover.GetCPUInfo,reschedDelay: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int) {+ loadFn: func(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList, numParallel int) {// add small delay to simulate loadingtime.Sleep(time.Millisecond)req.successCh <- &runnerRef{diff --git a/server/sched.go b/server/sched.goindex 3c8656ada..1409ff07f 100644--- a/server/sched.go+++ b/server/sched.go@@ -15,9 +15,9 @@ import ("time""github.com/ollama/ollama/api"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format"- "github.com/ollama/ollama/gpu""github.com/ollama/ollama/llm")@@ -41,10 +41,10 @@ type Scheduler struct {loaded map[string]*runnerRefloadedMu sync.Mutex- loadFn func(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int)- newServerFn func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error)- getGpuFn func() gpu.GpuInfoList- getCpuFn func() gpu.GpuInfoList+ loadFn func(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList, numParallel int)+ newServerFn func(gpus discover.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error)+ getGpuFn func() discover.GpuInfoList+ getCpuFn func() discover.GpuInfoListreschedDelay time.Duration}@@ -69,8 +69,8 @@ func InitScheduler(ctx context.Context) *Scheduler {unloadedCh: make(chan interface{}, maxQueue),loaded: make(map[string]*runnerRef),newServerFn: llm.NewLlamaServer,- getGpuFn: gpu.GetGPUInfo,- getCpuFn: gpu.GetCPUInfo,+ getGpuFn: discover.GetGPUInfo,+ getCpuFn: discover.GetCPUInfo,reschedDelay: 250 * time.Millisecond,}sched.loadFn = sched.load@@ -157,7 +157,7 @@ func (s *Scheduler) processPending(ctx context.Context) {} else {// Either no models are loaded or below envconfig.MaxRunners// Get a refreshed GPU list- var gpus gpu.GpuInfoList+ var gpus discover.GpuInfoListif pending.opts.NumGPU == 0 {gpus = s.getCpuFn()} else {@@ -409,7 +409,7 @@ func (pending *LlmRequest) useLoadedRunner(runner *runnerRef, finished chan *Llm}()}-func (s *Scheduler) load(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel int) {+func (s *Scheduler) load(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList, numParallel int) {if numParallel < 1 {numParallel = 1}@@ -470,7 +470,7 @@ func (s *Scheduler) load(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList,}()}-func (s *Scheduler) updateFreeSpace(allGpus gpu.GpuInfoList) {+func (s *Scheduler) updateFreeSpace(allGpus discover.GpuInfoList) {type predKey struct {Library stringID string@@ -513,8 +513,8 @@ func (s *Scheduler) updateFreeSpace(allGpus gpu.GpuInfoList) {// to avoid scheduling another model on the same GPU(s) that haven't stabilized.// This routine returns the set of GPUs that do not have an active loading model.// If all GPUs have loading models, an empty list will be returned (not a single CPU entry)-func (s *Scheduler) filterGPUsWithoutLoadingModels(allGpus gpu.GpuInfoList) gpu.GpuInfoList {- ret := append(gpu.GpuInfoList{}, allGpus...)+func (s *Scheduler) filterGPUsWithoutLoadingModels(allGpus discover.GpuInfoList) discover.GpuInfoList {+ ret := append(discover.GpuInfoList{}, allGpus...)s.loadedMu.Lock()defer s.loadedMu.Unlock()for _, runner := range s.loaded {@@ -541,8 +541,8 @@ type runnerRef struct {// unloading bool // set to true when we are trying to unload the runnerllama llm.LlamaServer- loading bool // True only during initial load, then false forever- gpus gpu.GpuInfoList // Recorded at time of provisioning+ loading bool // True only during initial load, then false forever+ gpus discover.GpuInfoList // Recorded at time of provisioningestimatedVRAM uint64estimatedTotal uint64@@ -630,7 +630,7 @@ func (runner *runnerRef) waitForVRAMRecovery() chan interface{} {start := time.Now()// Establish a baseline before we unload- gpusBefore := gpu.GetGPUInfo()+ gpusBefore := discover.GetGPUInfo()var totalMemoryBefore, freeMemoryBefore uint64for _, gpu := range gpusBefore {totalMemoryBefore += gpu.TotalMemory@@ -648,7 +648,7 @@ func (runner *runnerRef) waitForVRAMRecovery() chan interface{} {}// Query GPUs, look for free to go back up- gpusNow := gpu.GetGPUInfo()+ gpusNow := discover.GetGPUInfo()var totalMemoryNow, freeMemoryNow uint64for _, gpu := range gpusNow {totalMemoryNow += gpu.TotalMemory@@ -685,7 +685,7 @@ func (a ByDuration) Less(i, j int) bool {// If the model can not be fit fully within the available GPU(s) nil is returned// If numParallel is <= 0, this will attempt try to optimize parallism based on available VRAM, and adjust// opts.NumCtx accordingly-func pickBestFullFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel *int) gpu.GpuInfoList {+func pickBestFullFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList, numParallel *int) discover.GpuInfoList {var estimatedVRAM uint64var numParallelToTry []int@@ -698,22 +698,22 @@ func pickBestFullFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoLfor _, gl := range gpus.ByLibrary() {var ok bool- sgl := append(make(gpu.GpuInfoList, 0, len(gl)), gl...)+ sgl := append(make(discover.GpuInfoList, 0, len(gl)), gl...)// TODO - potentially sort by performance capability, existing models loaded, etc.// TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them// Note: at present, this will favor more VRAM over faster GPU speed in mixed setups- sort.Sort(sort.Reverse(gpu.ByFreeMemory(sgl)))+ sort.Sort(sort.Reverse(discover.ByFreeMemory(sgl)))// First attempt to fit the model into a single GPUfor _, p := range numParallelToTry {req.opts.NumCtx = req.origNumCtx * pif !envconfig.SchedSpread() {for _, g := range sgl {- if ok, estimatedVRAM = llm.PredictServerFit([]gpu.GpuInfo{g}, ggml, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts); ok {+ if ok, estimatedVRAM = llm.PredictServerFit([]discover.GpuInfo{g}, ggml, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts); ok {slog.Info("new model will fit in available VRAM in single GPU, loading", "model", req.model.ModelPath, "gpu", g.ID, "parallel", p, "available", g.FreeMemory, "required", format.HumanBytes2(estimatedVRAM))*numParallel = p- return []gpu.GpuInfo{g}+ return []discover.GpuInfo{g}}}}@@ -737,7 +737,7 @@ func pickBestFullFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoL}// If multiple Libraries are detected, pick the Library which loads the most layers for the model-func pickBestPartialFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel *int) gpu.GpuInfoList {+func pickBestPartialFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList, numParallel *int) discover.GpuInfoList {if *numParallel <= 0 {*numParallel = 1req.opts.NumCtx = req.origNumCtx@@ -822,7 +822,7 @@ func (s *Scheduler) expireRunner(model *Model) {// If other runners are loaded, make sure the pending request will fit in system memory// If not, pick a runner to unload, else return nil and the request can be loaded-func (s *Scheduler) maybeFindCPURunnerToUnload(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList) *runnerRef {+func (s *Scheduler) maybeFindCPURunnerToUnload(req *LlmRequest, ggml *llm.GGML, gpus discover.GpuInfoList) *runnerRef {slog.Debug("evaluating if CPU model load will fit in available system memory")estimate := llm.EstimateGPULayers(gpus, ggml, req.model.ProjectorPaths, req.opts)if estimate.TotalSize <= gpus[0].FreeMemory {diff --git a/server/sched_test.go b/server/sched_test.goindex fe5647c52..c999eee05 100644--- a/server/sched_test.go+++ b/server/sched_test.go@@ -13,8 +13,8 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/app/lifecycle"+ "github.com/ollama/ollama/discover""github.com/ollama/ollama/format"- "github.com/ollama/ollama/gpu""github.com/ollama/ollama/llm")@@ -47,10 +47,10 @@ func TestLoad(t *testing.T) {sessionDuration: &api.Duration{Duration: 2 * time.Second},}// Fail to load model first- s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(gpus discover.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {return nil, errors.New("something failed to load model blah")}- gpus := gpu.GpuInfoList{}+ gpus := discover.GpuInfoList{}s.load(req, ggml, gpus, 0)require.Empty(t, req.successCh)require.Len(t, req.errCh, 1)@@ -61,7 +61,7 @@ func TestLoad(t *testing.T) {require.Contains(t, err.Error(), "this model may be incompatible")server := &mockLlm{estimatedVRAM: 10, estimatedVRAMByGPU: map[string]uint64{}}- s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(gpus discover.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {return server, nil}s.load(req, ggml, gpus, 0)@@ -102,7 +102,7 @@ type reqBundle struct {ggml *llm.GGML}-func (scenario *reqBundle) newServer(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+func (scenario *reqBundle) newServer(gpus discover.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {return scenario.srv, nil}@@ -151,18 +151,18 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, estreturn b}-func getGpuFn() gpu.GpuInfoList {- g := gpu.GpuInfo{Library: "metal"}+func getGpuFn() discover.GpuInfoList {+ g := discover.GpuInfo{Library: "metal"}g.TotalMemory = 24 * format.GigaByteg.FreeMemory = 12 * format.GigaByte- return []gpu.GpuInfo{g}+ return []discover.GpuInfo{g}}-func getCpuFn() gpu.GpuInfoList {- g := gpu.GpuInfo{Library: "cpu"}+func getCpuFn() discover.GpuInfoList {+ g := discover.GpuInfo{Library: "cpu"}g.TotalMemory = 32 * format.GigaByteg.FreeMemory = 26 * format.GigaByte- return []gpu.GpuInfo{g}+ return []discover.GpuInfo{g}}func TestRequestsSameModelSameRequest(t *testing.T) {@@ -420,9 +420,9 @@ func TestExpireRunner(t *testing.T) {}var ggml *llm.GGML- gpus := gpu.GpuInfoList{}+ gpus := discover.GpuInfoList{}server := &mockLlm{estimatedVRAM: 10, estimatedVRAMByGPU: map[string]uint64{}}- s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(gpus discover.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {return server, nil}s.load(req, ggml, gpus, 0)@@ -460,11 +460,11 @@ func TestPrematureExpired(t *testing.T) {// Same model, same requestscenario1a := newScenarioRequest(t, ctx, "ollama-model-1a", 10, nil)s := InitScheduler(ctx)- s.getGpuFn = func() gpu.GpuInfoList {- g := gpu.GpuInfo{Library: "metal"}+ s.getGpuFn = func() discover.GpuInfoList {+ g := discover.GpuInfo{Library: "metal"}g.TotalMemory = 24 * format.GigaByteg.FreeMemory = 12 * format.GigaByte- return []gpu.GpuInfo{g}+ return []discover.GpuInfo{g}}s.newServerFn = scenario1a.newServersuccessCh1a, errCh1a := s.GetRunner(scenario1a.ctx, scenario1a.req.model, scenario1a.req.opts, scenario1a.req.sessionDuration)@@ -530,7 +530,7 @@ func TestUseLoadedRunner(t *testing.T) {func TestUpdateFreeSpace(t *testing.T) {ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)defer done()- gpus := gpu.GpuInfoList{+ gpus := discover.GpuInfoList{{Library: "a",ID: "1",@@ -563,7 +563,7 @@ func TestUpdateFreeSpace(t *testing.T) {func TestFilterGPUsWithoutLoadingModels(t *testing.T) {ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)defer done()- gpus := gpu.GpuInfoList{+ gpus := discover.GpuInfoList{{Library: "cuda",ID: "0",@@ -573,7 +573,7 @@ func TestFilterGPUsWithoutLoadingModels(t *testing.T) {ID: "1",},}- r1 := &runnerRef{gpus: gpu.GpuInfoList{gpus[0]}, loading: true}+ r1 := &runnerRef{gpus: discover.GpuInfoList{gpus[0]}, loading: true}s := InitScheduler(ctx)s.loadedMu.Lock()@@ -584,12 +584,12 @@ func TestFilterGPUsWithoutLoadingModels(t *testing.T) {require.Len(t, tmp, 1)require.Equal(t, "1", tmp[0].ID)- r1.gpus = gpu.GpuInfoList{gpus[1]}+ r1.gpus = discover.GpuInfoList{gpus[1]}tmp = s.filterGPUsWithoutLoadingModels(gpus)require.Len(t, tmp, 1)require.Equal(t, "0", tmp[0].ID)- r1.gpus = gpu.GpuInfoList{}+ r1.gpus = discover.GpuInfoList{}tmp = s.filterGPUsWithoutLoadingModels(gpus)require.Len(t, tmp, 2)}@@ -715,9 +715,9 @@ func TestHomogeneousGPUs(t *testing.T) {defer done()s := InitScheduler(ctx)- s.getGpuFn = func() gpu.GpuInfoList {+ s.getGpuFn = func() discover.GpuInfoList {// Set memory values to require the model to be spread- gpus := []gpu.GpuInfo{+ gpus := []discover.GpuInfo{{Library: "cuda"},{Library: "rocm"},}@@ -729,7 +729,7 @@ func TestHomogeneousGPUs(t *testing.T) {}s.getCpuFn = getCpuFna := newScenarioRequest(t, ctx, "ollama-model-1", 10, &api.Duration{Duration: 5 * time.Millisecond})- s.newServerFn = func(gpus gpu.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(gpus discover.GpuInfoList, model string, ggml *llm.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {require.Len(t, gpus, 1)return a.newServer(gpus, model, ggml, adapters, projectors, opts, numParallel)}