Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
DRY out the runner lifecycle code (#12540)
Daniel Hiltgen3258a89b6e4c
diff --git a/discover/cpu_linux_test.go b/discover/cpu_linux_test.goindex 3a5144780..7ff34df0d 100644--- a/discover/cpu_linux_test.go+++ b/discover/cpu_linux_test.go@@ -2065,12 +2065,6 @@ power management:cpus := linuxCPUDetails(buf)slog.Info("example", "scenario", k, "cpus", cpus)- si := SystemInfo{- System: CPUInfo{- CPUs: cpus,- },- }- threadCount := si.GetOptimalThreadCount()if len(v.expCPUs) != len(cpus) {t.Fatalf("incorrect number of sockets: expected:%v got:%v", v.expCPUs, cpus)}@@ -2085,10 +2079,6 @@ power management:t.Fatalf("incorrect number of threads: expected:%v got:%v", v.expCPUs[i], c)}}-- if threadCount != v.expThreadCount {- t.Fatalf("incorrect thread count expected:%d got:%d", v.expThreadCount, threadCount)- }})}}diff --git a/discover/gpu.go b/discover/gpu.goindex 2f394fdf8..927aed2a2 100644--- a/discover/gpu.go+++ b/discover/gpu.go@@ -1,16 +1,13 @@package discoverimport (- "context""log/slog""os"- "path/filepath""regexp""runtime""strconv""strings"- "github.com/ollama/ollama/format""github.com/ollama/ollama/ml")@@ -18,159 +15,28 @@ import (// Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.var CudaTegra string = os.Getenv("JETSON_JETPACK")-func GetCPUInfo() GpuInfo {- mem, err := GetCPUMem()+// GetSystemInfo returns the last cached state of the GPUs on the system+func GetSystemInfo() ml.SystemInfo {+ memInfo, err := GetCPUMem()if err != nil {slog.Warn("error looking up system memory", "error", err)}-- return GpuInfo{- memInfo: mem,- DeviceID: ml.DeviceID{- Library: "cpu",- ID: "0",- },- }-}--func GetGPUInfo(ctx context.Context, runners []FilteredRunnerDiscovery) GpuInfoList {- devs := GPUDevices(ctx, runners)- return devInfoToInfoList(devs)-}--func devInfoToInfoList(devs []ml.DeviceInfo) GpuInfoList {- resp := []GpuInfo{}- // Our current packaging model places ggml-hip in the main directory- // but keeps rocm in an isolated directory. We have to add it to- // the [LD_LIBRARY_]PATH so ggml-hip will load properly- rocmDir := filepath.Join(LibOllamaPath, "rocm")- if _, err := os.Stat(rocmDir); err != nil {- rocmDir = ""- }-- for _, dev := range devs {- info := GpuInfo{- DeviceID: dev.DeviceID,- filterID: dev.FilteredID,- Name: dev.Description,- memInfo: memInfo{- TotalMemory: dev.TotalMemory,- FreeMemory: dev.FreeMemory,- },- // TODO can we avoid variant- DependencyPath: dev.LibraryPath,- DriverMajor: dev.DriverMajor,- DriverMinor: dev.DriverMinor,- ComputeMajor: dev.ComputeMajor,- ComputeMinor: dev.ComputeMinor,- }- if dev.Library == "CUDA" || dev.Library == "ROCm" {- info.MinimumMemory = 457 * format.MebiByte- }- if dev.Library == "ROCm" && rocmDir != "" {- info.DependencyPath = append(info.DependencyPath, rocmDir)- }- // TODO any special processing of Vulkan devices?- resp = append(resp, info)- }- if len(resp) == 0 {- mem, err := GetCPUMem()- if err != nil {- slog.Warn("error looking up system memory", "error", err)- }-- resp = append(resp, GpuInfo{- memInfo: mem,- DeviceID: ml.DeviceID{- Library: "cpu",- ID: "0",- },- })+ var threadCount int+ cpus := GetCPUDetails()+ for _, c := range cpus {+ threadCount += c.CoreCount - c.EfficiencyCoreCount}- return resp-}-// Given the list of GPUs this instantiation is targeted for,-// figure out the visible devices environment variable-//-// If different libraries are detected, the first one is what we use-func (l GpuInfoList) GetVisibleDevicesEnv() []string {- if len(l) == 0 {- return nil- }- res := []string{}- envVar := rocmGetVisibleDevicesEnv(l)- if envVar != "" {- res = append(res, envVar)- }- envVar = vkGetVisibleDevicesEnv(l)- if envVar != "" {- res = append(res, envVar)- }- return res-}--func rocmGetVisibleDevicesEnv(gpuInfo []GpuInfo) string {- ids := []string{}- for _, info := range gpuInfo {- if info.Library != "ROCm" {- continue- }- // If the devices requires a numeric ID, for filtering purposes, we use the unfiltered ID number- if info.filterID != "" {- ids = append(ids, info.filterID)- } else {- ids = append(ids, info.ID)- }- }- if len(ids) == 0 {- return ""- }- envVar := "ROCR_VISIBLE_DEVICES="- if runtime.GOOS != "linux" {- envVar = "HIP_VISIBLE_DEVICES="- }- // There are 3 potential env vars to use to select GPUs.- // ROCR_VISIBLE_DEVICES supports UUID or numeric but does not work on Windows- // HIP_VISIBLE_DEVICES supports numeric IDs only- // GPU_DEVICE_ORDINAL supports numeric IDs only- return envVar + strings.Join(ids, ",")-}--func vkGetVisibleDevicesEnv(gpuInfo []GpuInfo) string {- ids := []string{}- for _, info := range gpuInfo {- if info.Library != "Vulkan" {- continue- }- if info.filterID != "" {- ids = append(ids, info.filterID)- } else {- ids = append(ids, info.ID)- }- }- if len(ids) == 0 {- return ""- }- envVar := "GGML_VK_VISIBLE_DEVICES="- return envVar + strings.Join(ids, ",")-}--// GetSystemInfo returns the last cached state of the GPUs on the system-func GetSystemInfo() SystemInfo {- deviceMu.Lock()- defer deviceMu.Unlock()- gpus := devInfoToInfoList(devices)- if len(gpus) == 1 && gpus[0].Library == "cpu" {- gpus = []GpuInfo{}+ if threadCount == 0 {+ // Fall back to Go's num CPU+ threadCount = runtime.NumCPU()}- return SystemInfo{- System: CPUInfo{- CPUs: GetCPUDetails(),- GpuInfo: GetCPUInfo(),- },- GPUs: gpus,+ return ml.SystemInfo{+ ThreadCount: threadCount,+ TotalMemory: memInfo.TotalMemory,+ FreeMemory: memInfo.FreeMemory,+ FreeSwap: memInfo.FreeSwap,}}diff --git a/discover/runner.go b/discover/runner.goindex 66c3e3e62..cbaba3c60 100644--- a/discover/runner.go+++ b/discover/runner.go@@ -4,13 +4,8 @@ package discoverimport ("context"- "encoding/json"- "fmt""io""log/slog"- "math/rand"- "net"- "net/http""os""os/exec""path/filepath"@@ -23,6 +18,7 @@ import ("github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format"+ "github.com/ollama/ollama/llm""github.com/ollama/ollama/logutil""github.com/ollama/ollama/ml")@@ -36,7 +32,7 @@ var (bootstrapped bool)-func GPUDevices(ctx context.Context, runners []FilteredRunnerDiscovery) []ml.DeviceInfo {+func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.DeviceInfo {deviceMu.Lock()defer deviceMu.Unlock()startDiscovery := time.Now()@@ -154,9 +150,9 @@ func GPUDevices(ctx context.Context, runners []FilteredRunnerDiscovery) []ml.Devslog.Error("Unknown Library:" + devices[i].Library)}- extraEnvs := []string{- "GGML_CUDA_INIT=1", // force deep initialization to trigger crash on unsupported GPUs- envVar + "=" + id, // Filter to just this one GPU+ extraEnvs := map[string]string{+ "GGML_CUDA_INIT": "1", // force deep initialization to trigger crash on unsupported GPUs+ envVar: id, // Filter to just this one GPU}if len(bootstrapDevices(ctx2ndPass, devices[i].LibraryPath, extraEnvs)) == 0 {needsDelete[i] = true@@ -449,100 +445,35 @@ func (r *bootstrapRunner) HasExited() bool {return false}-func bootstrapDevices(ctx context.Context, ollamaLibDirs []string, extraEnvs []string) []ml.DeviceInfo {- // TODO DRY out with llm/server.go- slog.Debug("spawning runner with", "OLLAMA_LIBRARY_PATH", ollamaLibDirs, "extra_envs", extraEnvs)+func bootstrapDevices(ctx context.Context, ollamaLibDirs []string, extraEnvs map[string]string) []ml.DeviceInfo {+ var out io.Writer+ if envconfig.LogLevel() == logutil.LevelTrace {+ out = os.Stderr+ }start := time.Now()defer func() {slog.Debug("bootstrap discovery took", "duration", time.Since(start), "OLLAMA_LIBRARY_PATH", ollamaLibDirs, "extra_envs", extraEnvs)}()- port := 0- if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {- var l *net.TCPListener- if l, err = net.ListenTCP("tcp", a); err == nil {- port = l.Addr().(*net.TCPAddr).Port- l.Close()- }- }- if port == 0 {- slog.Debug("ResolveTCPAddr failed, using random port")- port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range- }- params := []string{"runner", "--ollama-engine", "--port", strconv.Itoa(port)}- var pathEnv string- switch runtime.GOOS {- case "windows":- pathEnv = "PATH"- case "darwin":- pathEnv = "DYLD_LIBRARY_PATH"- default:- pathEnv = "LD_LIBRARY_PATH"- }- libraryPaths := append([]string{LibOllamaPath}, ollamaLibDirs...)- if rocmDir != "" {- libraryPaths = append(libraryPaths, rocmDir)- }- // Note: we always put our dependency paths first- // since these are the exact version we compiled/linked against- if libraryPath, ok := os.LookupEnv(pathEnv); ok {- libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)- }-- cmd := exec.Command(exe, params...)- cmd.Env = os.Environ()- if envconfig.LogLevel() == logutil.LevelTrace {- cmd.Stdout = os.Stdout- cmd.Stderr = os.Stderr- }- // cmd.SysProcAttr = llm.LlamaServerSysProcAttr // circular dependency - bring back once refactored- pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))- pathNeeded := true- ollamaPathNeeded := true- extraDone := make([]bool, len(extraEnvs))- for i := range cmd.Env {- cmp := strings.SplitN(cmd.Env[i], "=", 2)- if strings.EqualFold(cmp[0], pathEnv) {- cmd.Env[i] = pathEnv + "=" + pathEnvVal- pathNeeded = false- } else if strings.EqualFold(cmp[0], "OLLAMA_LIBRARY_PATH") {- cmd.Env[i] = "OLLAMA_LIBRARY_PATH=" + strings.Join(ollamaLibDirs, string(filepath.ListSeparator))- ollamaPathNeeded = false- } else {- for j := range extraEnvs {- if extraDone[j] {- continue- }- extra := strings.SplitN(extraEnvs[j], "=", 2)- if cmp[0] == extra[0] {- cmd.Env[i] = extraEnvs[j]- extraDone[j] = true- }- }- }- }- if pathNeeded {- cmd.Env = append(cmd.Env, pathEnv+"="+pathEnvVal)- }- if ollamaPathNeeded {- cmd.Env = append(cmd.Env, "OLLAMA_LIBRARY_PATH="+strings.Join(ollamaLibDirs, string(filepath.ListSeparator)))- }- for i := range extraDone {- if !extraDone[i] {- cmd.Env = append(cmd.Env, extraEnvs[i])- }- }- logutil.Trace("starting runner for device discovery", "env", cmd.Env, "cmd", cmd)- if err := cmd.Start(); err != nil {- slog.Warn("unable to start discovery subprocess", "cmd", cmd, "error", err)+ logutil.Trace("starting runner for device discovery", "libDirs", ollamaLibDirs, "extraEnvs", extraEnvs)+ cmd, port, err := llm.StartRunner(+ true, // ollama engine+ "", // no model+ ollamaLibDirs,+ out,+ extraEnvs,+ )+ if err != nil {+ slog.Debug("failed to start runner to discovery GPUs", "error", err)return nil}+go func() {cmd.Wait() // exit status ignored}()defer cmd.Process.Kill()- devices, err := GetDevicesFromRunner(ctx, &bootstrapRunner{port: port, cmd: cmd})+ devices, err := ml.GetDevicesFromRunner(ctx, &bootstrapRunner{port: port, cmd: cmd})if err != nil {if cmd.ProcessState != nil && cmd.ProcessState.ExitCode() >= 0 {// Expected during bootstrapping while we filter out unsupported AMD GPUs@@ -555,52 +486,3 @@ func bootstrapDevices(ctx context.Context, ollamaLibDirs []string, extraEnvs []sreturn devices}--func GetDevicesFromRunner(ctx context.Context, runner BaseRunner) ([]ml.DeviceInfo, error) {- var moreDevices []ml.DeviceInfo- port := runner.GetPort()- tick := time.Tick(10 * time.Millisecond)- for {- select {- case <-ctx.Done():- return nil, fmt.Errorf("failed to finish discovery before timeout")- case <-tick:- r, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/info", port), nil)- if err != nil {- return nil, fmt.Errorf("failed to create request: %w", err)- }- r.Header.Set("Content-Type", "application/json")-- resp, err := http.DefaultClient.Do(r)- if err != nil {- // slog.Warn("failed to send request", "error", err)- if runner.HasExited() {- return nil, fmt.Errorf("runner crashed")- }- continue- }- defer resp.Body.Close()-- if resp.StatusCode == http.StatusNotFound {- // old runner, fall back to bootstrapping model- return nil, fmt.Errorf("llamarunner free vram reporting not supported")- }-- body, err := io.ReadAll(resp.Body)- if err != nil {- slog.Warn("failed to read response", "error", err)- continue- }- if resp.StatusCode != 200 {- logutil.Trace("runner failed to discover free VRAM", "status", resp.StatusCode, "response", body)- return nil, fmt.Errorf("runner error: %s", string(body))- }-- if err := json.Unmarshal(body, &moreDevices); err != nil {- slog.Warn("unmarshal encode response", "error", err)- continue- }- return moreDevices, nil- }- }-}diff --git a/discover/types.go b/discover/types.goindex adb2f43a7..b34bafd2d 100644--- a/discover/types.go+++ b/discover/types.go@@ -1,10 +1,8 @@package discoverimport (- "context""log/slog""path/filepath"- "runtime""strings""github.com/ollama/ollama/format"@@ -17,50 +15,6 @@ type memInfo struct {FreeSwap uint64 `json:"free_swap,omitempty"` // TODO split this out for system only}-// Beginning of an `ollama info` command-type GpuInfo struct { // TODO better name maybe "InferenceProcessor"?- ml.DeviceID- memInfo-- // Optional variant to select (e.g. versions, cpu feature flags)- Variant string `json:"variant"`-- // MinimumMemory represents the minimum memory required to use the GPU- MinimumMemory uint64 `json:"-"`-- // Any extra PATH/LD_LIBRARY_PATH dependencies required for the Library to operate properly- DependencyPath []string `json:"lib_path,omitempty"`-- // Set to true if we can NOT reliably discover FreeMemory. A value of true indicates- // the FreeMemory is best effort, and may over or under report actual memory usage- // False indicates FreeMemory can generally be trusted on this GPU- UnreliableFreeMemory bool-- // GPU information- filterID string // AMD/Vulkan Workaround: The numeric ID of the device used to filter out other devices- Name string `json:"name"` // user friendly name if available- ComputeMajor int `json:"compute_major"` // Compute Capability or gfx- ComputeMinor int `json:"compute_minor"`-- // Driver Information - TODO no need to put this on each GPU- DriverMajor int `json:"driver_major,omitempty"`- DriverMinor int `json:"driver_minor,omitempty"`-- // TODO other performance capability info to help in scheduling decisions-}--func (gpu GpuInfo) RunnerName() string {- if gpu.Variant != "" {- return gpu.Library + "_" + gpu.Variant- }- return gpu.Library-}--type CPUInfo struct {- GpuInfo- CPUs []CPU-}-// CPU type represents a CPU Package occupying a sockettype CPU struct {ID string `cpuinfo:"processor"`@@ -71,32 +25,6 @@ type CPU struct {ThreadCount int}-type GpuInfoList []GpuInfo--func (l GpuInfoList) ByLibrary() []GpuInfoList {- resp := []GpuInfoList{}- libs := []string{}- for _, info := range l {- found := false- requested := info.Library- if info.Variant != "" {- requested += "_" + info.Variant- }- for i, lib := range libs {- if lib == requested {- resp[i] = append(resp[i], info)- found = true- break- }- }- if !found {- libs = append(libs, requested)- resp = append(resp, []GpuInfo{info})- }- }- return resp-}-func LogDetails(devices []ml.DeviceInfo) {for _, dev := range devices {var libs []string@@ -141,74 +69,3 @@ func LogDetails(devices []ml.DeviceInfo) {)}}--// Sort by Free Space-type ByFreeMemory []GpuInfo--func (a ByFreeMemory) Len() int { return len(a) }-func (a ByFreeMemory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }-func (a ByFreeMemory) Less(i, j int) bool { return a[i].FreeMemory < a[j].FreeMemory }--type SystemInfo struct {- System CPUInfo `json:"system"`- GPUs []GpuInfo `json:"gpus"`-}--// Return the optimal number of threads to use for inference-func (si SystemInfo) GetOptimalThreadCount() int {- if len(si.System.CPUs) == 0 {- // Fall back to Go's num CPU- return runtime.NumCPU()- }-- coreCount := 0- for _, c := range si.System.CPUs {- coreCount += c.CoreCount - c.EfficiencyCoreCount- }-- return coreCount-}--// For each GPU, check if it does NOT support flash attention-func (l GpuInfoList) FlashAttentionSupported() bool {- for _, gpu := range l {- supportsFA := gpu.Library == "cpu" ||- gpu.Name == "Metal" || gpu.Library == "Metal" ||- (gpu.Library == "CUDA" && gpu.DriverMajor >= 7 && !(gpu.ComputeMajor == 7 && gpu.ComputeMinor == 2)) || // We don't have kernels for Jetson Xavier- gpu.Library == "ROCm" ||- gpu.Library == "Vulkan"-- if !supportsFA {- return false- }- }- return true-}--type BaseRunner interface {- // GetPort returns the localhost port number the runner is running on- GetPort() int-- // HasExited indicates if the runner is no longer running. This can be used during- // bootstrap to detect if a given filtered device is incompatible and triggered an assert- HasExited() bool-}--type RunnerDiscovery interface {- BaseRunner-- // GetDeviceInfos will perform a query of the underlying device libraries- // for device identification and free VRAM information- // During bootstrap scenarios, this routine may take seconds to complete- GetDeviceInfos(ctx context.Context) []ml.DeviceInfo-}--type FilteredRunnerDiscovery interface {- RunnerDiscovery-- // GetActiveDeviceIDs returns the filtered set of devices actively in- // use by this runner for running models. If the runner is a bootstrap runner, no devices- // will be active yet so no device IDs are returned.- // This routine will not query the underlying device and will return immediately- GetActiveDeviceIDs() []ml.DeviceID-}diff --git a/llm/memory.go b/llm/memory.goindex aa4927f16..15558109f 100644--- a/llm/memory.go+++ b/llm/memory.go@@ -4,27 +4,28 @@ import ("fmt""log/slog""os"+ "slices""sort""strings""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format""github.com/ollama/ollama/fs/ggml"+ "github.com/ollama/ollama/ml")// pickBestFullFitByLibrary will try to find the optimal placement of the model in the available GPUs where the model fully fits// The list of GPUs returned will always be the same brand (library)// If the model can not be fit fully within the available GPU(s) nil is returned-func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, gpus discover.GpuInfoList, numParallel int) discover.GpuInfoList {- for _, gl := range gpus.ByLibrary() {- sgl := append(make(discover.GpuInfoList, 0, len(gl)), gl...)+func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, gpus []ml.DeviceInfo, numParallel int) []ml.DeviceInfo {+ for _, gl := range ml.ByLibrary(gpus) {+ sgl := append(make([]ml.DeviceInfo, 0, len(gl)), gl...)// TODO - potentially sort by performance capability, existing models loaded, etc.// TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them// Note: at present, this will favor most current available VRAM descending and ignoring faster GPU speed in mixed setups- sort.Sort(sort.Reverse(discover.ByFreeMemory(sgl)))+ sort.Sort(sort.Reverse(ml.ByFreeMemory(sgl)))if !envconfig.SchedSpread() {// Try to pack into as few GPUs as possible, starting from 1 GPU@@ -63,8 +64,8 @@ func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []strin}// If multiple Libraries are detected, pick the Library which loads the most layers for the model-func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []string, opts api.Options, gpus discover.GpuInfoList, numParallel int) discover.GpuInfoList {- byLibrary := gpus.ByLibrary()+func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []string, opts api.Options, gpus []ml.DeviceInfo, numParallel int) []ml.DeviceInfo {+ byLibrary := ml.ByLibrary(gpus)if len(byLibrary) <= 1 {return gpus}@@ -81,10 +82,10 @@ func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []s}// This algorithm looks for a complete fit to determine if we need to unload other models-func predictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64) {+func predictServerFit(allGpus []ml.DeviceInfo, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64) {// Split up the GPUs by type and try themvar estimatedVRAM uint64- for _, gpus := range allGpus.ByLibrary() {+ for _, gpus := range ml.ByLibrary(allGpus) {var layerCount intestimate := estimateGPULayers(gpus, f, projectors, opts, numParallel)layerCount, estimatedVRAM = estimate.Layers, estimate.VRAMSize@@ -97,14 +98,23 @@ func predictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projreturn true, estimatedVRAM}}-- if len(gpus) == 1 && gpus[0].Library == "cpu" && estimate.TotalSize <= gpus[0].FreeMemory {- return true, estimatedVRAM- }}return false, estimatedVRAM}+func verifyCPUFit(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, systemInfo ml.SystemInfo, numParallel int) bool {+ estimate := estimateGPULayers(nil, f, projectors, opts, numParallel)+ if estimate.TotalSize > systemInfo.FreeMemory {+ return false+ }+ slog.Info("new model will fit in available system memory for CPU inference, loading",+ "model", modelPath,+ "parallel", numParallel,+ "required", format.HumanBytes2(estimate.TotalSize),+ )+ return true+}+type MemoryEstimate struct {// How many layers we predict we can loadLayers int@@ -141,7 +151,7 @@ type MemoryEstimate struct {// Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size// The GPUs provided must all be the same Library-func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []string, opts api.Options, numParallel int) MemoryEstimate {+func estimateGPULayers(gpus []ml.DeviceInfo, f *ggml.GGML, projectors []string, opts api.Options, numParallel int) MemoryEstimate {// Graph size for a partial offload, applies to all GPUsvar graphPartialOffload uint64@@ -175,10 +185,17 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinoverhead := envconfig.GpuOverhead()availableList := make([]string, len(gpus))+ libraries := []string{}for i, gpu := range gpus {availableList[i] = format.HumanBytes2(gpu.FreeMemory)+ if !slices.Contains(libraries, gpu.Library) {+ libraries = append(libraries, gpu.Library)+ }}- slog.Debug("evaluating", "library", gpus[0].Library, "gpu_count", len(gpus), "available", availableList)+ if len(libraries) == 0 {+ libraries = []string{"cpu"}+ }+ slog.Debug("evaluating", "library", strings.Join(libraries, ","), "gpu_count", len(gpus), "available", availableList)for _, projector := range projectors {llamaEngineProjectorWeights += projectorMemoryRequirements(projector)@@ -196,7 +213,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin}useFlashAttention := envconfig.FlashAttention(f.FlashAttention()) &&- (discover.GpuInfoList)(gpus).FlashAttentionSupported() &&+ ml.FlashAttentionSupported(gpus) &&f.SupportsFlashAttention()var kvct string@@ -231,7 +248,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin}// on metal there's no partial offload overhead- if gpus[0].Library == "Metal" {+ if len(gpus) > 0 && gpus[0].Library == "Metal" {graphPartialOffload = graphFullOffload} else if len(gpus) > 1 {// multigpu should always use the partial graph size@@ -256,7 +273,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []stringpuAllocations := make([]uint64, len(gpus))type gs struct {i int- g *discover.GpuInfo+ g *ml.DeviceInfo}gpusWithSpace := []gs{}for i := range gpus {@@ -265,19 +282,11 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []stringzo = gpuZeroOverhead}// Only include GPUs that can fit the graph, gpu minimum, the layer buffer and at least more layer- if gpus[i].FreeMemory < overhead+gzo+max(graphPartialOffload, graphFullOffload)+gpus[i].MinimumMemory+2*layerSize {- var compute string- if gpus[i].Library == "ROCm" {- compute = fmt.Sprintf("gfx%x%02x", gpus[i].ComputeMajor, gpus[i].ComputeMinor)- } else {- compute = fmt.Sprintf("%d.%d", gpus[i].ComputeMajor, gpus[i].ComputeMinor)- }-+ if gpus[i].FreeMemory < overhead+gzo+max(graphPartialOffload, graphFullOffload)+gpus[i].MinimumMemory()+2*layerSize {slog.Debug("gpu has too little memory to allocate any layers","id", gpus[i].ID,"library", gpus[i].Library,- "variant", gpus[i].Variant,- "compute", compute,+ "compute", gpus[i].Compute(),"driver", fmt.Sprintf("%d.%d", gpus[i].DriverMajor, gpus[i].DriverMinor),"name", gpus[i].Name,"total", format.HumanBytes2(gpus[i].TotalMemory),@@ -291,7 +300,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strincontinue}gpusWithSpace = append(gpusWithSpace, gs{i, &gpus[i]})- gpuAllocations[i] += gpus[i].MinimumMemory + layerSize // We hold off on graph until we know partial vs. full+ gpuAllocations[i] += gpus[i].MinimumMemory() + layerSize // We hold off on graph until we know partial vs. full}var gpuZeroID int@@ -397,7 +406,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinVRAMSize: 0,GPUSizes: []uint64{},- inferenceLibrary: gpus[0].Library,+ inferenceLibrary: strings.Join(libraries, ","),layersRequested: opts.NumGPU,layersModel: int(f.KV().BlockCount()) + 1,availableList: availableList,@@ -411,7 +420,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinprojectorGraph: ollamaEngineProjectorGraph,}- if gpus[0].Library == "cpu" {+ if len(gpus) == 0 {return estimate}if layerCount == 0 {diff --git a/llm/memory_test.go b/llm/memory_test.goindex 553214b9e..fce17b9c2 100644--- a/llm/memory_test.go+++ b/llm/memory_test.go@@ -10,7 +10,7 @@ import ("github.com/stretchr/testify/require""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover"+ "github.com/ollama/ollama/format""github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/ml")@@ -54,13 +54,7 @@ func TestEstimateGPULayers(t *testing.T) {}// Simple CPU scenario- gpus := []discover.GpuInfo{- {- DeviceID: ml.DeviceID{- Library: "cpu",- },- },- }+ gpus := []ml.DeviceInfo{}projectors := []string{}opts := api.DefaultOptions()t.Run("cpu", func(t *testing.T) {@@ -77,19 +71,17 @@ func TestEstimateGPULayers(t *testing.T) {memoryLayerOutput := uint64(4)// Dual CUDA scenario with asymmetry- gpuMinimumMemory := uint64(2048)- gpus = []discover.GpuInfo{+ gpuMinimumMemory := uint64(457 * format.MebiByte)+ gpus = []ml.DeviceInfo{{DeviceID: ml.DeviceID{- Library: "cuda",+ Library: "CUDA",},- MinimumMemory: gpuMinimumMemory,},{DeviceID: ml.DeviceID{- Library: "cuda",+ Library: "CUDA",},- MinimumMemory: gpuMinimumMemory,},}// Nested array: GPU0 layer space, GPU1 layer space, expected gpu0, expected gpu1diff --git a/llm/server.go b/llm/server.goindex 6ba8f8d2f..8e91e411e 100644--- a/llm/server.go+++ b/llm/server.go@@ -27,7 +27,6 @@ import ("golang.org/x/sync/semaphore""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format""github.com/ollama/ollama/fs/ggml"@@ -66,7 +65,7 @@ func (e filteredEnv) LogValue() slog.Value {type LlamaServer interface {ModelPath() string- Load(ctx context.Context, gpus discover.GpuInfoList, requireFull bool) ([]ml.DeviceID, error)+ Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error)Ping(ctx context.Context) errorWaitUntilRunning(ctx context.Context) errorCompletion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error@@ -115,7 +114,7 @@ type llamaServer struct {llmServerggml *ggml.GGML- gpus discover.GpuInfoList // The set of GPUs covered by the memory estimate+ gpus []ml.DeviceInfo // The set of GPUs covered by the memory estimateestimate MemoryEstimate}@@ -146,7 +145,7 @@ func LoadModel(model string, maxArraySize int) (*ggml.GGML, error) {}// NewLlamaServer will run a server for the given GPUs-func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {+func NewLlamaServer(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, modelPath string, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {var llamaModel *llama.Modelvar textProcessor model.TextProcessorvar err error@@ -179,7 +178,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, aloadRequest := LoadRequest{LoraPath: adapters, KvSize: opts.NumCtx * numParallel, BatchSize: opts.NumBatch, Parallel: numParallel, MultiUserCache: envconfig.MultiUserCache()}- defaultThreads := discover.GetSystemInfo().GetOptimalThreadCount()+ defaultThreads := systemInfo.ThreadCountif opts.NumThread > 0 {loadRequest.NumThreads = opts.NumThread} else if defaultThreads > 0 {@@ -200,7 +199,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a// This will disable flash attention unless all GPUs on the system support it, even if we end up selecting a subset// that can handle it.- if fa && !gpus.FlashAttentionSupported() {+ if fa && !ml.FlashAttentionSupported(gpus) {slog.Warn("flash attention enabled but not supported by gpu")fa = false}@@ -227,218 +226,170 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, aslog.Warn("quantized kv cache requested but flash attention disabled", "type", kvct)}- availableLibs := make(map[string]string)- if entries, err := os.ReadDir(discover.LibOllamaPath); err == nil {- for _, entry := range entries {- availableLibs[entry.Name()] = filepath.Join(discover.LibOllamaPath, entry.Name())- }- }-- var gpuLibs []string- for _, gpu := range gpus {- gpuLibs = append(gpuLibs, gpu.RunnerName())+ gpuLibs := ml.LibraryPaths(gpus)+ status := NewStatusWriter(os.Stderr)+ cmd, port, err := StartRunner(+ textProcessor != nil,+ modelPath,+ gpuLibs,+ status,+ ml.GetVisibleDevicesEnv(gpus),+ )++ s := llmServer{+ port: port,+ cmd: cmd,+ status: status,+ options: opts,+ modelPath: modelPath,+ loadRequest: loadRequest,+ llamaModel: llamaModel,+ llamaModelLock: &sync.Mutex{},+ textProcessor: textProcessor,+ numParallel: numParallel,+ sem: semaphore.NewWeighted(int64(numParallel)),+ totalLayers: f.KV().BlockCount() + 1,+ loadStart: time.Now(),+ done: make(chan error, 1),}- requested := envconfig.LLMLibrary()- if availableLibs[requested] != "" {- slog.Info("using requested gpu library", "requested", requested)- gpuLibs = []string{requested}+ if err != nil {+ var msg string+ if s.status != nil && s.status.LastErrMsg != "" {+ msg = s.status.LastErrMsg+ }+ err := fmt.Errorf("error starting runner: %v %s", err, msg)+ if llamaModel != nil {+ llama.FreeModel(llamaModel)+ }+ return nil, err}- var compatible []string- for _, gpuLib := range gpuLibs {- var matchingLibs []string- for k := range availableLibs {- // exact match first- if k == gpuLib {- matchingLibs = append([]string{k}, matchingLibs...)- continue- }-- // then match the family (e.g. 'cuda')- if strings.Split(k, "_")[0] == strings.Split(gpuLib, "_")[0] {- matchingLibs = append(matchingLibs, k)+ // reap subprocess when it exits+ go func() {+ err := s.cmd.Wait()+ // Favor a more detailed message over the process exit status+ if err != nil && s.status != nil && s.status.LastErrMsg != "" {+ slog.Error("llama runner terminated", "error", err)+ if strings.Contains(s.status.LastErrMsg, "unknown model") {+ s.status.LastErrMsg = "this model is not supported by your version of Ollama. You may need to upgrade"}+ s.done <- errors.New(s.status.LastErrMsg)+ } else {+ s.done <- err}+ }()- if len(matchingLibs) > 0 {- compatible = append(compatible, matchingLibs[0])- }+ if textProcessor != nil {+ return &ollamaServer{llmServer: s}, nil+ } else {+ return &llamaServer{llmServer: s, ggml: f}, nil}+}- exe, err := os.Executable()+func StartRunner(ollamaEngine bool, modelPath string, gpuLibs []string, out io.Writer, extraEnvs map[string]string) (cmd *exec.Cmd, port int, err error) {+ var exe string+ exe, err = os.Executable()if err != nil {- return nil, fmt.Errorf("unable to lookup executable path: %w", err)+ return nil, 0, fmt.Errorf("unable to lookup executable path: %w", err)}if eval, err := filepath.EvalSymlinks(exe); err == nil {exe = eval}- // iterate through compatible GPU libraries such as 'cuda_v12', 'rocm', etc.- // adding each library's respective path to the LD_LIBRARY_PATH, until finally running- // without any LD_LIBRARY_PATH flags- for {- port := 0- if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {- var l *net.TCPListener- if l, err = net.ListenTCP("tcp", a); err == nil {- port = l.Addr().(*net.TCPAddr).Port- l.Close()- }- }- if port == 0 {- slog.Debug("ResolveTCPAddr failed, using random port")- port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range- }- params := []string{"runner"}- if textProcessor != nil {- // New engine- // TODO - if we have failure to load scenarios, add logic to retry with the old runner- params = append(params, "--ollama-engine")+ port = 0+ if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {+ var l *net.TCPListener+ if l, err = net.ListenTCP("tcp", a); err == nil {+ port = l.Addr().(*net.TCPAddr).Port+ l.Close()}+ }+ if port == 0 {+ slog.Debug("ResolveTCPAddr failed, using random port")+ port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range+ }+ params := []string{"runner"}+ if ollamaEngine {+ params = append(params, "--ollama-engine")+ }+ if modelPath != "" {params = append(params, "--model", modelPath)- params = append(params, "--port", strconv.Itoa(port))-- var pathEnv string- switch runtime.GOOS {- case "windows":- pathEnv = "PATH"- case "darwin":- pathEnv = "DYLD_LIBRARY_PATH"- default:- pathEnv = "LD_LIBRARY_PATH"- }-- // Note: we always put our dependency paths first- // since these are the exact version we compiled/linked against- libraryPaths := []string{discover.LibOllamaPath}- if libraryPath, ok := os.LookupEnv(pathEnv); ok {- libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)- }-- ggmlPaths := []string{discover.LibOllamaPath}- for _, c := range compatible {- if libpath, ok := availableLibs[c]; ok {- slog.Debug("adding gpu library", "path", libpath)- libraryPaths = append([]string{libpath}, libraryPaths...)- ggmlPaths = append(ggmlPaths, libpath)- }- }-- for _, gpu := range gpus {- if gpu.DependencyPath != nil {- slog.Debug("adding gpu dependency paths", "paths", gpu.DependencyPath)- libraryPaths = append(gpu.DependencyPath, libraryPaths...)- ggmlPaths = append(ggmlPaths, gpu.DependencyPath...)- }- }+ }+ params = append(params, "--port", strconv.Itoa(port))- // finally, add the root library path- libraryPaths = append(libraryPaths, discover.LibOllamaPath)-- s := llmServer{- port: port,- cmd: exec.Command(exe, params...),- status: NewStatusWriter(os.Stderr),- options: opts,- modelPath: modelPath,- loadRequest: loadRequest,- llamaModel: llamaModel,- llamaModelLock: &sync.Mutex{},- textProcessor: textProcessor,- numParallel: numParallel,- sem: semaphore.NewWeighted(int64(numParallel)),- totalLayers: f.KV().BlockCount() + 1,- loadStart: time.Now(),- done: make(chan error, 1),- }-- s.cmd.Env = os.Environ()- s.cmd.Stdout = os.Stdout- s.cmd.Stderr = s.status- s.cmd.SysProcAttr = LlamaServerSysProcAttr-- // Always filter down the set of GPUs in case there are any unsupported devices that might crash- envWorkarounds := gpus.GetVisibleDevicesEnv()- pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))-- // Update or add the path variable with our adjusted version- pathNeeded := true- ollamaPathNeeded := true- envWorkaroundDone := make([]bool, len(envWorkarounds))- for i := range s.cmd.Env {- cmp := strings.SplitN(s.cmd.Env[i], "=", 2)- if strings.EqualFold(cmp[0], pathEnv) {- s.cmd.Env[i] = pathEnv + "=" + pathEnvVal- pathNeeded = false- } else if strings.EqualFold(cmp[0], "OLLAMA_LIBRARY_PATH") {- s.cmd.Env[i] = "OLLAMA_LIBRARY_PATH=" + strings.Join(ggmlPaths, string(filepath.ListSeparator))- ollamaPathNeeded = false- } else if len(envWorkarounds) != 0 {- for j, kv := range envWorkarounds {- tmp := strings.SplitN(kv, "=", 2)- if strings.EqualFold(cmp[0], tmp[0]) {- s.cmd.Env[i] = kv- envWorkaroundDone[j] = true- }+ var pathEnv string+ switch runtime.GOOS {+ case "windows":+ pathEnv = "PATH"+ case "darwin":+ pathEnv = "DYLD_LIBRARY_PATH"+ default:+ pathEnv = "LD_LIBRARY_PATH"+ }++ // Note: we always put our dependency paths first+ // since these are the exact version we compiled/linked against+ libraryPaths := append([]string{}, gpuLibs...)+ if libraryPath, ok := os.LookupEnv(pathEnv); ok {+ libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)+ }++ cmd = exec.Command(exe, params...)++ cmd.Env = os.Environ()+ cmd.Stdout = out+ cmd.Stderr = out+ cmd.SysProcAttr = LlamaServerSysProcAttr++ // Always filter down the set of GPUs in case there are any unsupported devices that might crash+ pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))++ // Update or add the path variable with our adjusted version+ pathNeeded := true+ ollamaPathNeeded := true+ extraEnvsDone := map[string]bool{}+ for k := range extraEnvs {+ extraEnvsDone[k] = false+ }+ for i := range cmd.Env {+ cmp := strings.SplitN(cmd.Env[i], "=", 2)+ if strings.EqualFold(cmp[0], pathEnv) {+ cmd.Env[i] = pathEnv + "=" + pathEnvVal+ pathNeeded = false+ } else if strings.EqualFold(cmp[0], "OLLAMA_LIBRARY_PATH") {+ cmd.Env[i] = "OLLAMA_LIBRARY_PATH=" + strings.Join(gpuLibs, string(filepath.ListSeparator))+ ollamaPathNeeded = false+ } else if len(extraEnvs) != 0 {+ for k, v := range extraEnvs {+ if strings.EqualFold(cmp[0], k) {+ cmd.Env[i] = k + "=" + v+ extraEnvsDone[k] = true}}}- if pathNeeded {- s.cmd.Env = append(s.cmd.Env, pathEnv+"="+pathEnvVal)- }- if ollamaPathNeeded {- s.cmd.Env = append(s.cmd.Env, "OLLAMA_LIBRARY_PATH="+strings.Join(ggmlPaths, string(filepath.ListSeparator)))- }- for i, done := range envWorkaroundDone {- if !done {- s.cmd.Env = append(s.cmd.Env, envWorkarounds[i])- }- }-- slog.Info("starting runner", "cmd", s.cmd)- slog.Debug("subprocess", "", filteredEnv(s.cmd.Env))-- if err = s.cmd.Start(); err != nil {- var msg string- if s.status != nil && s.status.LastErrMsg != "" {- msg = s.status.LastErrMsg- }- err := fmt.Errorf("error starting runner: %v %s", err, msg)- if len(compatible) == 0 {- if llamaModel != nil {- llama.FreeModel(llamaModel)- }- return nil, err- }-- slog.Warn("unable to start runner with compatible gpu", "error", err, "compatible", compatible)- compatible = compatible[1:]- continue+ }+ if pathNeeded {+ cmd.Env = append(cmd.Env, pathEnv+"="+pathEnvVal)+ }+ if ollamaPathNeeded {+ cmd.Env = append(cmd.Env, "OLLAMA_LIBRARY_PATH="+strings.Join(gpuLibs, string(filepath.ListSeparator)))+ }+ for k, done := range extraEnvsDone {+ if !done {+ cmd.Env = append(cmd.Env, k+"="+extraEnvs[k])}+ }- // reap subprocess when it exits- go func() {- err := s.cmd.Wait()- // Favor a more detailed message over the process exit status- if err != nil && s.status != nil && s.status.LastErrMsg != "" {- slog.Error("llama runner terminated", "error", err)- if strings.Contains(s.status.LastErrMsg, "unknown model") {- s.status.LastErrMsg = "this model is not supported by your version of Ollama. You may need to upgrade"- }- s.done <- errors.New(s.status.LastErrMsg)- } else {- s.done <- err- }- }()+ slog.Info("starting runner", "cmd", cmd)+ slog.Debug("subprocess", "", filteredEnv(cmd.Env))- if textProcessor != nil {- return &ollamaServer{llmServer: s}, nil- } else {- return &llamaServer{llmServer: s, ggml: f}, nil- }+ if err = cmd.Start(); err != nil {+ return nil, 0, err}+ err = nil+ return}func (s *llmServer) ModelPath() string {@@ -497,47 +448,58 @@ type LoadResponse struct {var ErrLoadRequiredFull = errors.New("unable to load full model on GPU")-func (s *llamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requireFull bool) ([]ml.DeviceID, error) {- systemInfo := discover.GetSystemInfo()- systemTotalMemory := systemInfo.System.TotalMemory- systemFreeMemory := systemInfo.System.FreeMemory- systemSwapFreeMemory := systemInfo.System.FreeSwap+func (s *llamaServer) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {+ systemTotalMemory := systemInfo.TotalMemory+ systemFreeMemory := systemInfo.FreeMemory+ systemSwapFreeMemory := systemInfo.FreeSwapslog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))- g := pickBestFullFitByLibrary(s.ggml, s.modelPath, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)- if g == nil {- if !requireFull {- g = pickBestPartialFitByLibrary(s.ggml, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)- } else {+ if len(gpus) == 0 || s.options.NumGPU == 0 {+ if !verifyCPUFit(s.ggml, s.modelPath, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, systemInfo, s.numParallel) {slog.Info("model requires more memory than is currently available, evicting a model to make space", "estimate", s.estimate)- return nil, ErrLoadRequiredFull+ return nil, fmt.Errorf("model requires more system memory than is currently available %w", ErrLoadRequiredFull)+ }+ } else {+ g := pickBestFullFitByLibrary(s.ggml, s.modelPath, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)+ if g == nil {+ if !requireFull {+ g = pickBestPartialFitByLibrary(s.ggml, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)+ } else {+ slog.Info("model requires more memory than is currently available, evicting a model to make space", "estimate", s.estimate)+ return nil, ErrLoadRequiredFull+ }}+ gpus = g}- gpus = gs.estimate = estimateGPULayers(gpus, s.ggml, []string{s.loadRequest.ProjectorPath}, s.options, s.numParallel)- if len(gpus) > 1 || gpus[0].Library != "cpu" {+ if len(gpus) >= 1 {switch {- case gpus[0].Library == "Metal" && s.estimate.VRAMSize > systemInfo.System.TotalMemory:+ case s.options.NumGPU == 0:+ gpus = []ml.DeviceInfo{}+ case gpus[0].Library == "Metal" && s.estimate.VRAMSize > systemInfo.TotalMemory:// disable partial offloading when model is greater than total system memory as this// can lead to locking up the systems.options.NumGPU = 0+ gpus = []ml.DeviceInfo{}case gpus[0].Library != "Metal" && s.estimate.Layers == 0:// Don't bother loading into the GPU if no layers can fit- gpus = discover.GpuInfoList{discover.GetCPUInfo()}- case s.options.NumGPU < 0 && s.estimate.Layers > 0 && gpus[0].Library != "cpu":+ gpus = []ml.DeviceInfo{}+ case s.options.NumGPU < 0 && s.estimate.Layers > 0:s.options.NumGPU = s.estimate.Layers}+ } else {+ s.options.NumGPU = 0}// On linux and windows, over-allocating CPU memory will almost always result in an error// Darwin has fully dynamic swap so has no direct concept of free swap spaceif runtime.GOOS != "darwin" {systemMemoryRequired := s.estimate.TotalSize - s.estimate.VRAMSize- available := systemInfo.System.FreeMemory + systemInfo.System.FreeSwap+ available := systemInfo.FreeMemory + systemInfo.FreeSwapif systemMemoryRequired > available {- slog.Warn("model request too large for system", "requested", format.HumanBytes2(systemMemoryRequired), "available", format.HumanBytes2(available), "total", format.HumanBytes2(systemInfo.System.TotalMemory), "free", format.HumanBytes2(systemInfo.System.FreeMemory), "swap", format.HumanBytes2(systemInfo.System.FreeSwap))+ slog.Warn("model request too large for system", "requested", format.HumanBytes2(systemMemoryRequired), "available", format.HumanBytes2(available), "total", format.HumanBytes2(systemInfo.TotalMemory), "free", format.HumanBytes2(systemInfo.FreeMemory), "swap", format.HumanBytes2(systemInfo.FreeSwap))return nil, fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(systemMemoryRequired), format.HumanBytes2(available))}}@@ -564,10 +526,10 @@ func (s *llamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requi// Windows CUDA should not use mmap for best performance// Linux with a model larger than free space, mmap leads to thrashing// For CPU loads we want the memory to be allocated, not FS cache- if (runtime.GOOS == "windows" && gpus[0].Library == "CUDA" && s.options.UseMMap == nil) ||- (runtime.GOOS == "linux" && systemInfo.System.FreeMemory < s.estimate.TotalSize && s.options.UseMMap == nil) ||- (gpus[0].Library == "cpu" && s.options.UseMMap == nil) ||- (gpus[0].Library == "Vulkan" && s.options.UseMMap == nil) ||+ if (runtime.GOOS == "windows" && len(gpus) > 0 && gpus[0].Library == "CUDA" && s.options.UseMMap == nil) ||+ (runtime.GOOS == "linux" && systemInfo.FreeMemory < s.estimate.TotalSize && s.options.UseMMap == nil) ||+ (len(gpus) == 0 && s.options.UseMMap == nil) ||+ (len(gpus) > 0 && gpus[0].Library == "Vulkan" && s.options.UseMMap == nil) ||(s.options.UseMMap != nil && !*s.options.UseMMap) {s.loadRequest.UseMmap = false}@@ -605,8 +567,8 @@ func (s *llamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requi// createGPULayers maps from the tensor splits assigned by the memory estimates to explicit assignment// of particular layers onto GPUs-func createGPULayers(estimate MemoryEstimate, ggml *ggml.GGML, gpus discover.GpuInfoList, numGPU int) ml.GPULayersList {- if numGPU <= 0 {+func createGPULayers(estimate MemoryEstimate, ggml *ggml.GGML, gpus []ml.DeviceInfo, numGPU int) ml.GPULayersList {+ if numGPU <= 0 || len(gpus) == 0 {return nil}@@ -662,7 +624,7 @@ func createGPULayers(estimate MemoryEstimate, ggml *ggml.GGML, gpus discover.Gpu// allowing for faster iteration, but may return less information.//// Returns the list of GPU IDs that were used in the final allocation on success-func (s *ollamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requireFull bool) ([]ml.DeviceID, error) {+func (s *ollamaServer) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {var success booldefer func() {if !success {@@ -675,24 +637,21 @@ func (s *ollamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requslog.Info("loading model", "model layers", s.totalLayers, "requested", s.options.NumGPU)- systemInfo := discover.GetSystemInfo()- systemTotalMemory := systemInfo.System.TotalMemory- systemFreeMemory := systemInfo.System.FreeMemory- systemSwapFreeMemory := systemInfo.System.FreeSwap+ systemTotalMemory := systemInfo.TotalMemory+ systemFreeMemory := systemInfo.FreeMemory+ systemSwapFreeMemory := systemInfo.FreeSwapslog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))- if !(len(gpus) == 1 && gpus[0].Library == "cpu") {- for _, gpu := range gpus {- available := gpu.FreeMemory - envconfig.GpuOverhead() - gpu.MinimumMemory- if gpu.FreeMemory < envconfig.GpuOverhead()+gpu.MinimumMemory {- available = 0- }- slog.Info("gpu memory", "id", gpu.ID, "library", gpu.Library,- "available", format.HumanBytes2(available),- "free", format.HumanBytes2(gpu.FreeMemory),- "minimum", format.HumanBytes2(gpu.MinimumMemory),- "overhead", format.HumanBytes2(envconfig.GpuOverhead()))+ for _, gpu := range gpus {+ available := gpu.FreeMemory - envconfig.GpuOverhead() - gpu.MinimumMemory()+ if gpu.FreeMemory < envconfig.GpuOverhead()+gpu.MinimumMemory() {+ available = 0}+ slog.Info("gpu memory", "id", gpu.ID, "library", gpu.Library,+ "available", format.HumanBytes2(available),+ "free", format.HumanBytes2(gpu.FreeMemory),+ "minimum", format.HumanBytes2(gpu.MinimumMemory()),+ "overhead", format.HumanBytes2(envconfig.GpuOverhead()))}pastAllocations := make(map[uint64]struct{})@@ -762,7 +721,6 @@ nextOperation:if err != nil {return nil, err}-slog.Debug("new layout created", "layers", newGPULayers)s.loadRequest.GPULayers = newGPULayers@@ -864,20 +822,27 @@ func uniqueDeviceIDs(gpuLayers ml.GPULayersList) []ml.DeviceID {// - Calculating how much space each GPU has available for layers, based on free memory and space occupied by the graph// - Assigning layers// - Ensuring that we don't exceed limits, such as requirements about partial offloading or system memory-func (s *ollamaServer) createLayout(systemInfo discover.SystemInfo, systemGPUs discover.GpuInfoList, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, error) {- if s.totalLayers == 0 || s.options.NumGPU == 0 || len(systemGPUs) == 0 || (len(systemGPUs) == 1 && systemGPUs[0].Library == "cpu") {- return ml.GPULayersList{}, nil- }-- gpus := append(make(discover.GpuInfoList, 0, len(systemGPUs)), systemGPUs...)- sort.Sort(sort.Reverse(discover.ByFreeMemory(gpus)))-+func (s *ollamaServer) createLayout(systemInfo ml.SystemInfo, systemGPUs []ml.DeviceInfo, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, error) {if memory == nil {memory = &ml.BackendMemory{CPU: ml.DeviceMemory{Weights: make([]uint64, s.totalLayers),Cache: make([]uint64, s.totalLayers),}}}+ gpuLayers, layers, err := s.buildLayout(systemGPUs, memory, requireFull, backoff)+ if err != nil {+ return nil, err+ }+ err = s.verifyLayout(systemInfo, memory, requireFull, gpuLayers, layers)+ if err != nil {+ return nil, err+ }+ return gpuLayers, nil+}++func (s *ollamaServer) buildLayout(systemGPUs []ml.DeviceInfo, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, []uint64, error) {+ gpus := append(make([]ml.DeviceInfo, 0, len(systemGPUs)), systemGPUs...)+ sort.Sort(sort.Reverse(ml.ByFreeMemory(gpus)))layers := make([]uint64, len(memory.CPU.Weights))for i := range layers {@@ -891,7 +856,7 @@ func (s *ollamaServer) createLayout(systemInfo discover.SystemInfo, systemGPUs d}gpuLayers := ml.GPULayersList{}- for _, gl := range gpus.ByLibrary() {+ for _, gl := range ml.ByLibrary(gpus) {// If a GPU already has a graph allocated on it, then we should continue to use it.// Otherwise, we lose information that we got from previous allocations, which can// cause cycling. Plus, we get more information about required allocation from each@@ -905,7 +870,7 @@ func (s *ollamaServer) createLayout(systemInfo discover.SystemInfo, systemGPUs dlastUsedGPU = i}- reserved := uint64(float32(gl[i].FreeMemory)*backoff) + gl[i].MinimumMemory + envconfig.GpuOverhead() + memory.GPUs[j].Graph+ reserved := uint64(float32(gl[i].FreeMemory)*backoff) + gl[i].MinimumMemory() + envconfig.GpuOverhead() + memory.GPUs[j].Graphif gl[i].FreeMemory > reserved {gl[i].FreeMemory -= reserved} else {@@ -914,7 +879,7 @@ func (s *ollamaServer) createLayout(systemInfo discover.SystemInfo, systemGPUs dslog.Debug("available gpu", "id", gl[i].ID, "library", gl[i].Library,"available layer vram", format.HumanBytes2(gl[i].FreeMemory),- "backoff", fmt.Sprintf("%.2f", backoff), "minimum", format.HumanBytes2(gl[i].MinimumMemory),+ "backoff", fmt.Sprintf("%.2f", backoff), "minimum", format.HumanBytes2(gl[i].MinimumMemory()),"overhead", format.HumanBytes2(envconfig.GpuOverhead()),"graph", format.HumanBytes2(memory.GPUs[j].Graph))@@ -933,7 +898,11 @@ func (s *ollamaServer) createLayout(systemInfo discover.SystemInfo, systemGPUs dgpuLayers = libraryGpuLayers}}+ return gpuLayers, layers, nil+}+// verifyLayout ensures that we don't exceed limits, such as requirements about partial offloading or system memory+func (s *ollamaServer) verifyLayout(systemInfo ml.SystemInfo, memory *ml.BackendMemory, requireFull bool, gpuLayers ml.GPULayersList, layers []uint64) error {// These sizes will only increase as we go through additional iterations and get additional information.cpuSize := memory.InputWeights + memory.CPU.Graphvar vramSize uint64@@ -961,24 +930,24 @@ nextLayer:if requireFull {if gpuLayers.Sum() < len(layers) && (s.options.NumGPU < 0 || gpuLayers.Sum() < s.options.NumGPU) {- return nil, ErrLoadRequiredFull+ return ErrLoadRequiredFull}- if cpuSize > systemInfo.System.FreeMemory {- return nil, ErrLoadRequiredFull+ if cpuSize > systemInfo.FreeMemory {+ return ErrLoadRequiredFull}}// On linux and windows, over-allocating CPU memory will almost always result in an error// Darwin has fully dynamic swap so has no direct concept of free swap spaceif runtime.GOOS != "darwin" {- available := systemInfo.System.FreeMemory + systemInfo.System.FreeSwap+ available := systemInfo.FreeMemory + systemInfo.FreeSwapif cpuSize > available {- slog.Warn("model request too large for system", "requested", format.HumanBytes2(cpuSize), "available", format.HumanBytes2(available), "total", format.HumanBytes2(systemInfo.System.TotalMemory), "free", format.HumanBytes2(systemInfo.System.FreeMemory), "swap", format.HumanBytes2(systemInfo.System.FreeSwap))- return nil, fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(cpuSize), format.HumanBytes2(available))+ slog.Warn("model request too large for system", "requested", format.HumanBytes2(cpuSize), "available", format.HumanBytes2(available), "total", format.HumanBytes2(systemInfo.TotalMemory), "free", format.HumanBytes2(systemInfo.FreeMemory), "swap", format.HumanBytes2(systemInfo.FreeSwap))+ return fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(cpuSize), format.HumanBytes2(available))}} else {- if vramSize > systemInfo.System.TotalMemory {+ if vramSize > systemInfo.TotalMemory {// disable partial offloading when model is greater than total system memory as this// can lead to locking up the systems.options.NumGPU = 0@@ -990,11 +959,11 @@ nextLayer:slog.Debug("insufficient VRAM to load any model layers")}- return gpuLayers, nil+ return nil}// assignLayers packs the maximum number of layers onto the smallest set of GPUs and comes up with a layer assignment-func assignLayers(layers []uint64, gpus discover.GpuInfoList, requireFull bool, requestedLayers int, lastUsedGPU int) (gpuLayers ml.GPULayersList) {+func assignLayers(layers []uint64, gpus []ml.DeviceInfo, requireFull bool, requestedLayers int, lastUsedGPU int) (gpuLayers ml.GPULayersList) {// If we can't fit everything then prefer offloading layers other than the output layerfor range 2 {// requestedLayers may be -1 if nothing was requested@@ -1028,7 +997,7 @@ func assignLayers(layers []uint64, gpus discover.GpuInfoList, requireFull bool,// findBestFit binary searches to find the smallest capacity factor that can fit// the max number of layers. The capacity factor is multiplied by the free space on// each GPU and a small one will force even balancing.-func findBestFit(layers []uint64, gpus discover.GpuInfoList, requestedLayers int, forceRequest bool) (gpuLayers ml.GPULayersList) {+func findBestFit(layers []uint64, gpus []ml.DeviceInfo, requestedLayers int, forceRequest bool) (gpuLayers ml.GPULayersList) {var high float32 = 1var low float32 = 0@@ -1053,12 +1022,11 @@ func findBestFit(layers []uint64, gpus discover.GpuInfoList, requestedLayers intlow = mid}}-return bestAssignments}// greedyFit assigns layers incrementally to GPUs, spilling over as each runs out of free space-func greedyFit(layers []uint64, gpus discover.GpuInfoList, capacity float32, requestedLayers int) (gpuLayers ml.GPULayersList) {+func greedyFit(layers []uint64, gpus []ml.DeviceInfo, capacity float32, requestedLayers int) (gpuLayers ml.GPULayersList) {device := len(gpus) - 1gpuLayers = ml.GPULayersList{{DeviceID: gpus[device].DeviceID}}freeSpace := uint64(float32(gpus[device].FreeMemory) * capacity)@@ -1082,7 +1050,6 @@ func greedyFit(layers []uint64, gpus discover.GpuInfoList, capacity float32, reqfreeSpace = uint64(float32(gpus[device].FreeMemory) * capacity)}}-return gpuLayers}@@ -1814,7 +1781,7 @@ func (s *ollamaServer) VRAMByGPU(id ml.DeviceID) uint64 {}func (s *ollamaServer) GetDeviceInfos(ctx context.Context) []ml.DeviceInfo {- devices, err := discover.GetDevicesFromRunner(ctx, s)+ devices, err := ml.GetDevicesFromRunner(ctx, s)if err != nil {if s.cmd != nil && s.cmd.ProcessState == nil {// Still running but hit an error, logdiff --git a/llm/server_test.go b/llm/server_test.goindex bdedc960e..2d3bf6bec 100644--- a/llm/server_test.go+++ b/llm/server_test.go@@ -8,7 +8,6 @@ import ("testing""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/format""github.com/ollama/ollama/ml""golang.org/x/sync/semaphore"@@ -20,6 +19,8 @@ func TestLLMServerFitGPU(t *testing.T) {free int}+ minMemory := 457 * format.MebiByte+tests := []struct {name stringgpus []gpu@@ -37,91 +38,91 @@ func TestLLMServerFitGPU(t *testing.T) {},{name: "Full single GPU",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256*format.MebiByte + minMemory}},layers: []int{50 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: -1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{0, 1, 2}}},},{name: "Partial single GPU",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256*format.MebiByte + minMemory}},layers: []int{100 * format.MebiByte, 100 * format.MebiByte, 100 * format.MebiByte, 100 * format.MebiByte},numGPU: -1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{1, 2}}},},{name: "Single GPU with numGPU 1",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256*format.MebiByte + minMemory}},layers: []int{50 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: 1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{1}}},},{name: "Single GPU with numGPU 0",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256*format.MebiByte + minMemory}},layers: []int{50 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: 0,expected: ml.GPULayersList{},},{name: "Single GPU with numGPU 999",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256*format.MebiByte + minMemory}},layers: []int{100 * format.MebiByte, 100 * format.MebiByte, 100 * format.MebiByte, 100 * format.MebiByte},numGPU: 999,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{0, 1, 2, 3}}},},{name: "Multi GPU fits on one",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{50 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: -1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1"}, Layers: []int{0, 1, 2}}},},{name: "Multi GPU split",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{256 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: -1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1"}, Layers: []int{0}}, {DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{1, 2}}},},{name: "Multi GPU partial",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{256 * format.MebiByte, 256 * format.MebiByte, 50 * format.MebiByte},numGPU: -1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1"}, Layers: []int{1}}},},{name: "Multi GPU numGPU 1",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{50 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: 1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1"}, Layers: []int{1}}},},{name: "Multi GPU numGPU 2",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{256 * format.MebiByte, 50 * format.MebiByte, 50 * format.MebiByte},numGPU: 2,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1"}, Layers: []int{0}}, {DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{1}}},},{name: "Multi GPU numGPU 999",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{256 * format.MebiByte, 256 * format.MebiByte, 50 * format.MebiByte},numGPU: 999,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1"}, Layers: []int{0, 1}}, {DeviceID: ml.DeviceID{ID: "gpu0"}, Layers: []int{2}}},},{name: "Multi GPU different libraries",- gpus: []gpu{{id: ml.DeviceID{Library: "CUDA", ID: "gpu0"}, free: 128 * format.MebiByte}, {id: ml.DeviceID{Library: "ROCm", ID: "gpu1"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{Library: "CUDA", ID: "gpu0"}, free: 128*format.MebiByte + minMemory}, {id: ml.DeviceID{Library: "ROCm", ID: "gpu1"}, free: 256*format.MebiByte + minMemory}},layers: []int{128 * format.MebiByte, 128 * format.MebiByte, 50 * format.MebiByte},numGPU: -1,expected: ml.GPULayersList{{DeviceID: ml.DeviceID{ID: "gpu1", Library: "ROCm"}, Layers: []int{0, 1}}},},{name: "requireFull",- gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256 * format.MebiByte}},+ gpus: []gpu{{id: ml.DeviceID{ID: "gpu0"}, free: 256*format.MebiByte + minMemory}},layers: []int{100 * format.MebiByte, 100 * format.MebiByte, 100 * format.MebiByte, 100 * format.MebiByte},numGPU: -1,requireFull: true,@@ -139,12 +140,12 @@ func TestLLMServerFitGPU(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- var systemInfo discover.SystemInfo- systemInfo.System.TotalMemory = format.GibiByte- systemInfo.System.FreeMemory = 512 * format.MebiByte- systemInfo.System.FreeSwap = 256 * format.MebiByte+ var systemInfo ml.SystemInfo+ systemInfo.TotalMemory = format.GibiByte+ systemInfo.FreeMemory = 512 * format.MebiByte+ systemInfo.FreeSwap = 256 * format.MebiByte- gpus := make(discover.GpuInfoList, len(tt.gpus))+ gpus := make([]ml.DeviceInfo, len(tt.gpus))for i := range tt.gpus {gpus[i].DeviceID = tt.gpus[i].idgpus[i].FreeMemory = uint64(tt.gpus[i].free)diff --git a/ml/device.go b/ml/device.goindex 6569d87bb..39fba7d17 100644--- a/ml/device.go+++ b/ml/device.go@@ -3,15 +3,21 @@ package mlimport ("context""encoding/binary"+ "encoding/json""fmt""hash/maphash"+ "io""log/slog"+ "net/http"+ "runtime""slices""sort""strconv""strings"+ "time""github.com/ollama/ollama/format"+ "github.com/ollama/ollama/logutil")// GPULayers is a set of layers to be allocated on a single GPU@@ -282,6 +288,20 @@ type DeviceInfo struct {LibraryPath []string}+type SystemInfo struct {+ // ThreadCount is the optimal number of threads to use for inference+ ThreadCount int `json:"threads,omitempty"`++ // TotalMemory is the total amount of system memory+ TotalMemory uint64 `json:"total_memory,omitempty"`++ // FreeMemory is the amount of memory currently available on the system for loading models+ FreeMemory uint64 `json:"free_memory,omitempty"`++ // FreeSwap is the amount of system swap space reported as available+ FreeSwap uint64 `json:"free_swap,omitempty"`+}+func (d DeviceInfo) Compute() string {// AMD gfx is encoded into the major minor in hex formif strings.EqualFold(d.Library, "ROCm") {@@ -294,6 +314,71 @@ func (d DeviceInfo) Driver() string {return strconv.Itoa(d.DriverMajor) + "." + strconv.Itoa(d.DriverMinor)}+// MinimumMemory reports the amount of memory that should be set aside+// on the device for overhead (e.g. VRAM consumed by context structures independent+// of model allocations)+func (d DeviceInfo) MinimumMemory() uint64 {+ if d.Library == "Metal" {+ return 512 * format.MebiByte+ }+ return 457 * format.MebiByte+}++// Sort by Free Space.+// iGPUs are reported first, thus Reverse() yields the largest discrete GPU first+type ByFreeMemory []DeviceInfo++func (a ByFreeMemory) Len() int { return len(a) }+func (a ByFreeMemory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }+func (a ByFreeMemory) Less(i, j int) bool {+ if a[i].Integrated && !a[j].Integrated {+ return true+ } else if !a[i].Integrated && a[j].Integrated {+ return false+ }+ return a[i].FreeMemory < a[j].FreeMemory+}++func ByLibrary(l []DeviceInfo) [][]DeviceInfo {+ resp := [][]DeviceInfo{}+ libs := []string{}+ for _, info := range l {+ found := false+ requested := info.Library+ for i, lib := range libs {+ if lib == requested {+ resp[i] = append(resp[i], info)+ found = true+ break+ }+ }+ if !found {+ libs = append(libs, requested)+ resp = append(resp, []DeviceInfo{info})+ }+ }+ return resp+}++func LibraryPaths(l []DeviceInfo) []string {+ var gpuLibs []string+ for _, gpu := range l {+ for _, dir := range gpu.LibraryPath {+ needed := true+ for _, existing := range gpuLibs {+ if dir == existing {+ needed = false+ break+ }+ }+ if needed {+ gpuLibs = append(gpuLibs, dir)+ }+ }+ }+ return gpuLibs+}+type DeviceComparison intconst (@@ -336,3 +421,133 @@ func (a DeviceInfo) IsBetter(b DeviceInfo) bool {sort.Sort(sort.Reverse(sort.StringSlice(cmp)))return cmp[0] == bLibSplit[1]}++// For each GPU, check if it does NOT support flash attention+func FlashAttentionSupported(l []DeviceInfo) bool {+ for _, gpu := range l {+ supportsFA := gpu.Library == "cpu" ||+ gpu.Name == "Metal" || gpu.Library == "Metal" ||+ (gpu.Library == "CUDA" && gpu.DriverMajor >= 7 && !(gpu.ComputeMajor == 7 && gpu.ComputeMinor == 2)) ||+ gpu.Library == "ROCm"++ if !supportsFA {+ return false+ }+ }+ return true+}++// Given the list of GPUs this instantiation is targeted for,+// figure out the visible devices environment variables+func GetVisibleDevicesEnv(l []DeviceInfo) map[string]string {+ if len(l) == 0 {+ return nil+ }+ env := map[string]string{}+ for _, d := range l {+ d.updateVisibleDevicesEnv(env)+ }+ return env+}++func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) {+ var envVar string+ switch d.Library {+ case "ROCm":+ envVar = "ROCR_VISIBLE_DEVICES"+ if runtime.GOOS != "linux" {+ envVar = "HIP_VISIBLE_DEVICES"+ }+ case "Vulkan":+ envVar = "GGML_VK_VISIBLE_DEVICES"+ default:+ return+ }+ v, existing := env[envVar]+ if existing {+ v = v + ","+ }+ if d.FilteredID != "" {+ v = v + d.FilteredID+ } else {+ v = v + d.ID+ }+ env[envVar] = v+}++type BaseRunner interface {+ // GetPort returns the localhost port number the runner is running on+ GetPort() int++ // HasExited indicates if the runner is no longer running. This can be used during+ // bootstrap to detect if a given filtered device is incompatible and triggered an assert+ HasExited() bool+}++type RunnerDiscovery interface {+ BaseRunner++ // GetDeviceInfos will perform a query of the underlying device libraries+ // for device identification and free VRAM information+ // During bootstrap scenarios, this routine may take seconds to complete+ GetDeviceInfos(ctx context.Context) []DeviceInfo+}++type FilteredRunnerDiscovery interface {+ RunnerDiscovery++ // GetActiveDeviceIDs returns the filtered set of devices actively in+ // use by this runner for running models. If the runner is a bootstrap runner, no devices+ // will be active yet so no device IDs are returned.+ // This routine will not query the underlying device and will return immediately+ GetActiveDeviceIDs() []DeviceID+}++func GetDevicesFromRunner(ctx context.Context, runner BaseRunner) ([]DeviceInfo, error) {+ var moreDevices []DeviceInfo+ port := runner.GetPort()+ tick := time.Tick(10 * time.Millisecond)+ for {+ select {+ case <-ctx.Done():+ return nil, fmt.Errorf("failed to finish discovery before timeout")+ case <-tick:+ r, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/info", port), nil)+ if err != nil {+ return nil, fmt.Errorf("failed to create request: %w", err)+ }+ r.Header.Set("Content-Type", "application/json")++ resp, err := http.DefaultClient.Do(r)+ if err != nil {+ // slog.Warn("failed to send request", "error", err)+ if runner.HasExited() {+ return nil, fmt.Errorf("runner crashed")+ }+ continue+ }+ defer resp.Body.Close()++ if resp.StatusCode == http.StatusNotFound {+ // old runner, fall back to bootstrapping model+ return nil, fmt.Errorf("llamarunner free vram reporting not supported")+ }++ body, err := io.ReadAll(resp.Body)+ if err != nil {+ slog.Warn("failed to read response", "error", err)+ continue+ }+ if resp.StatusCode != 200 {+ logutil.Trace("runner failed to discover free VRAM", "status", resp.StatusCode, "response", body)+ return nil, fmt.Errorf("runner error: %s", string(body))+ }++ if err := json.Unmarshal(body, &moreDevices); err != nil {+ slog.Warn("unmarshal encode response", "error", err)+ continue+ }+ return moreDevices, nil+ }+ }+}diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1index f1cd3fea2..548545cb7 100644--- a/scripts/build_windows.ps1+++ b/scripts/build_windows.ps1@@ -84,11 +84,11 @@ function buildCPU() {Remove-Item -ea 0 -recurse -force -path "${script:SRC_DIR}\dist\windows-${script:ARCH}"New-Item "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\" -ItemType Directory -ea 0- & cmake --fresh --preset CPU --install-prefix $script:DIST_DIR+ & cmake -B build\cpu --preset CPU --install-prefix $script:DIST_DIRif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --build --preset CPU --config Release --parallel $script:JOBS+ & cmake --build build\cpu --target ggml-cpu --config Release --parallel $script:JOBSif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --install build --component CPU --strip+ & cmake --install build\cpu --component CPU --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}}}@@ -105,11 +105,11 @@ function buildCUDA11() {$hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V11")) { $x=$hashEnv[$_]; if (test-path -literalpath "$x\bin\nvcc.exe" ) { $cuda=$x} }}write-host "Building CUDA v11 backend libraries $cuda"$env:CUDAToolkit_ROOT=$cuda- & cmake --fresh --preset "CUDA 11" -T cuda="$cuda" -DCMAKE_CUDA_COMPILER="$cuda\bin\nvcc.exe" -G "Visual Studio 16 2019" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v11"+ & cmake -B build\cuda_v11 --preset "CUDA 11" -T cuda="$cuda" -DCMAKE_CUDA_COMPILER="$cuda\bin\nvcc.exe" -G "Visual Studio 16 2019" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v11"if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --build --preset "CUDA 11" --config Release --parallel $script:JOBS+ & cmake --build build\cuda_v11 --target ggml-cuda --config Release --parallel $script:JOBSif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --install build --component "CUDA" --strip+ & cmake --install build\cuda_v11 --component "CUDA" --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}}}@@ -124,11 +124,11 @@ function buildCUDA12() {$hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V12_8")) { $x=$hashEnv[$_]; if (test-path -literalpath "$x\bin\nvcc.exe" ) { $cuda=$x} }}write-host "Building CUDA v12 backend libraries $cuda"$env:CUDAToolkit_ROOT=$cuda- & cmake --fresh --preset "CUDA 12" -T cuda="$cuda" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v12"+ & cmake -B build\cuda_v12 --preset "CUDA 12" -T cuda="$cuda" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v12"if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --build --preset "CUDA 12" --config Release --parallel $script:JOBS+ & cmake --build build\cuda_v12 --target ggml-cuda --config Release --parallel $script:JOBSif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --install build --component "CUDA" --strip+ & cmake --install build\cuda_v12 --component "CUDA" --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}}}@@ -143,11 +143,11 @@ function buildCUDA13() {$hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V13")) { $x=$hashEnv[$_]; if (test-path -literalpath "$x\bin\nvcc.exe" ) { $cuda=$x} }}$env:CUDAToolkit_ROOT=$cudawrite-host "Building CUDA v13 backend libraries $cuda"- & cmake --fresh --preset "CUDA 13" -T cuda="$cuda" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v13"+ & cmake -B build\cuda_v13 --preset "CUDA 13" -T cuda="$cuda" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v13"if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --build --preset "CUDA 13" --config Release --parallel $script:JOBS+ & cmake --build build\cuda_v13 --target ggml-cuda --config Release --parallel $script:JOBSif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --install build --component "CUDA" --strip+ & cmake --install build\cuda_v13 --component "CUDA" --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}}}@@ -165,7 +165,7 @@ function buildROCm() {$env:HIPCXX="${env:HIP_PATH}\bin\clang++.exe"$env:HIP_PLATFORM="amd"$env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"- & cmake --fresh --preset "ROCm 6" -G Ninja -DOLLAMA_RUNNER_DIR="rocm" `+ & cmake --fresh -B build\rocm --preset "ROCm 6" -G Ninja -DOLLAMA_RUNNER_DIR="rocm" `-DCMAKE_C_COMPILER=clang `-DCMAKE_CXX_COMPILER=clang++ `-DCMAKE_C_FLAGS="-parallel-jobs=4 -Wno-ignored-attributes -Wno-deprecated-pragma" `@@ -175,9 +175,9 @@ function buildROCm() {$env:HIPCXX=""$env:HIP_PLATFORM=""$env:CMAKE_PREFIX_PATH=""- & cmake --build --preset "ROCm 6" --config Release --parallel $script:JOBS+ & cmake --build build\rocm --target ggml-hip --config Release --parallel $script:JOBSif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}- & cmake --install build --component "HIP" --strip+ & cmake --install build\rocm --component "HIP" --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}Remove-Item -Path $script:DIST_DIR\lib\ollama\rocm\rocblas\library\*gfx906* -ErrorAction SilentlyContinue}diff --git a/server/routes_debug_test.go b/server/routes_debug_test.goindex 466951a1d..bf822c68b 100644--- a/server/routes_debug_test.go+++ b/server/routes_debug_test.go@@ -9,9 +9,9 @@ import ("github.com/gin-gonic/gin""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/ml")func TestGenerateDebugRenderOnly(t *testing.T) {@@ -37,9 +37,9 @@ func TestGenerateDebugRenderOnly(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {// add small delay to simulate loadingtime.Sleep(time.Millisecond)req.successCh <- &runnerRef{@@ -230,9 +230,9 @@ func TestChatDebugRenderOnly(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {// add small delay to simulate loadingtime.Sleep(time.Millisecond)req.successCh <- &runnerRef{diff --git a/server/routes_generate_renderer_test.go b/server/routes_generate_renderer_test.goindex ea18b1e55..e6473e087 100644--- a/server/routes_generate_renderer_test.go+++ b/server/routes_generate_renderer_test.go@@ -12,9 +12,9 @@ import ("github.com/google/go-cmp/cmp""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/ml")// TestGenerateWithBuiltinRenderer tests that api/generate uses built-in renderers@@ -42,9 +42,9 @@ func TestGenerateWithBuiltinRenderer(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {time.Sleep(time.Millisecond)req.successCh <- &runnerRef{llama: &mock,@@ -226,9 +226,9 @@ func TestGenerateWithDebugRenderOnly(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {time.Sleep(time.Millisecond)req.successCh <- &runnerRef{llama: &mock,diff --git a/server/routes_generate_test.go b/server/routes_generate_test.goindex bcd774fd1..4c6b934b3 100644--- a/server/routes_generate_test.go+++ b/server/routes_generate_test.go@@ -17,9 +17,9 @@ import ("github.com/google/go-cmp/cmp""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/ml")type mockRunner struct {@@ -48,8 +48,8 @@ func (mockRunner) Tokenize(_ context.Context, s string) (tokens []int, err errorreturn}-func newMockServer(mock *mockRunner) func(discover.GpuInfoList, string, *ggml.GGML, []string, []string, api.Options, int) (llm.LlamaServer, error) {- return func(_ discover.GpuInfoList, _ string, _ *ggml.GGML, _, _ []string, _ api.Options, _ int) (llm.LlamaServer, error) {+func newMockServer(mock *mockRunner) func(ml.SystemInfo, []ml.DeviceInfo, string, *ggml.GGML, []string, []string, api.Options, int) (llm.LlamaServer, error) {+ return func(_ ml.SystemInfo, _ []ml.DeviceInfo, _ string, _ *ggml.GGML, _, _ []string, _ api.Options, _ int) (llm.LlamaServer, error) {return mock, nil}}@@ -157,9 +157,9 @@ func TestGenerateChat(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {// add small delay to simulate loadingtime.Sleep(time.Millisecond)req.successCh <- &runnerRef{@@ -768,9 +768,9 @@ func TestGenerate(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {// add small delay to simulate loadingtime.Sleep(time.Millisecond)req.successCh <- &runnerRef{@@ -1193,9 +1193,9 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {time.Sleep(time.Millisecond)req.successCh <- &runnerRef{llama: mock}return falsediff --git a/server/routes_harmony_streaming_test.go b/server/routes_harmony_streaming_test.goindex caf2cf6d5..8e58ad962 100644--- a/server/routes_harmony_streaming_test.go+++ b/server/routes_harmony_streaming_test.go@@ -14,9 +14,9 @@ import ("github.com/gin-gonic/gin""github.com/ollama/ollama/api"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/ml")func getTestTools() []api.Tool {@@ -275,9 +275,9 @@ func TestChatHarmonyParserStreamingRealtime(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 100 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {req.successCh <- &runnerRef{llama: &mock,}@@ -426,9 +426,9 @@ func TestChatHarmonyParserStreamingSimple(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 100 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {req.successCh <- &runnerRef{llama: &mock,}@@ -608,9 +608,9 @@ func TestChatHarmonyParserStreaming(t *testing.T) {loaded: make(map[string]*runnerRef),newServerFn: newMockServer(&mock),getGpuFn: getGpuFn,- getCpuFn: getCpuFn,+ getSystemInfoFn: getSystemInfoFn,waitForRecovery: 250 * time.Millisecond,- loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ bool) bool {+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {req.successCh <- &runnerRef{llama: &mock,}diff --git a/server/sched.go b/server/sched.goindex 7c6399531..e262d26fd 100644--- a/server/sched.go+++ b/server/sched.go@@ -5,12 +5,9 @@ import ("errors""fmt""log/slog"- "os""reflect"- "runtime""slices""sort"- "strconv""strings""sync""time"@@ -52,12 +49,10 @@ type Scheduler struct {activeLoading llm.LlamaServerloaded map[string]*runnerRef- loadFn func(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoList, requireFull bool) bool- newServerFn func(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error)- getGpuFn func(ctx context.Context, runners []discover.FilteredRunnerDiscovery) discover.GpuInfoList- getCpuFn func() discover.GpuInfo-- // waitForRecovery sets the limit for how long to wait for memory usage to recover after unload before scheduling the next model+ loadFn func(req *LlmRequest, f *ggml.GGML, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) bool+ newServerFn func(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error)+ getGpuFn func(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.DeviceInfo+ getSystemInfoFn func() ml.SystemInfowaitForRecovery time.Duration}@@ -77,8 +72,8 @@ func InitScheduler(ctx context.Context) *Scheduler {unloadedCh: make(chan any, maxQueue),loaded: make(map[string]*runnerRef),newServerFn: llm.NewLlamaServer,- getGpuFn: discover.GetGPUInfo,- getCpuFn: discover.GetCPUInfo,+ getGpuFn: discover.GPUDevices,+ getSystemInfoFn: discover.GetSystemInfo,waitForRecovery: 5 * time.Second,}sched.loadFn = sched.load@@ -133,6 +128,8 @@ func (s *Scheduler) Run(ctx context.Context) {}func (s *Scheduler) processPending(ctx context.Context) {+ maxRunners := envconfig.MaxRunners()+for {select {case <-ctx.Done():@@ -152,7 +149,7 @@ func (s *Scheduler) processPending(ctx context.Context) {s.loadedMu.Lock()runner := s.loaded[pending.model.ModelPath]loadedCount := len(s.loaded)- runnersSnapshot := make([]discover.FilteredRunnerDiscovery, 0, len(s.loaded))+ runnersSnapshot := make([]ml.FilteredRunnerDiscovery, 0, len(s.loaded))for _, r := range s.loaded {runnersSnapshot = append(runnersSnapshot, r)}@@ -167,39 +164,29 @@ func (s *Scheduler) processPending(ctx context.Context) {pending.useLoadedRunner(runner, s.finishedReqCh)break}- } else if envconfig.MaxRunners() > 0 && loadedCount >= int(envconfig.MaxRunners()) {+ } else if maxRunners > 0 && loadedCount >= int(maxRunners) {slog.Debug("max runners achieved, unloading one to make room", "runner_count", loadedCount)runnerToExpire = s.findRunnerToUnload()} else {// Either no models are loaded or below envconfig.MaxRunners// Get a refreshed GPU list- var gpus discover.GpuInfoList+ var gpus []ml.DeviceInfoif pending.opts.NumGPU == 0 {- gpus = discover.GpuInfoList{s.getCpuFn()}+ gpus = []ml.DeviceInfo{}} else {gpus = s.getGpuFn(ctx, runnersSnapshot)}-- if envconfig.MaxRunners() <= 0 {- // No user specified MaxRunners, so figure out what automatic setting to use- // If all GPUs have reliable free memory reporting, defaultModelsPerGPU * the number of GPUs- // if any GPU has unreliable free memory reporting, 1x the number of GPUs- allReliable := true- for _, gpu := range gpus {- if gpu.UnreliableFreeMemory {- allReliable = false- break- }- }- if allReliable {- // HACK- os.Setenv("OLLAMA_MAX_LOADED_MODELS", strconv.Itoa(defaultModelsPerGPU*len(gpus)))- slog.Debug("updating default concurrency", "OLLAMA_MAX_LOADED_MODELS", envconfig.MaxRunners(), "gpu_count", len(gpus))+ systemInfo := s.getSystemInfoFn()+ if maxRunners <= 0 {+ // No user specified MaxRunners, so figure out what automatic setting to use for the next load attempt+ if pending.opts.NumGPU == 0 {+ // Need to get actual GPU list to set the correct default max models+ g := s.getGpuFn(ctx, runnersSnapshot)+ maxRunners = uint(defaultModelsPerGPU * max(len(g), 1))} else {- // HACK- os.Setenv("OLLAMA_MAX_LOADED_MODELS", strconv.Itoa(len(gpus)))- slog.Info("one or more GPUs detected that are unable to accurately report free memory - disabling default concurrency")+ maxRunners = uint(defaultModelsPerGPU * max(len(gpus), 1))}+ slog.Debug("updating default concurrency", "OLLAMA_MAX_LOADED_MODELS", maxRunners, "gpu_count", len(gpus))}// Load model for fitting@@ -215,14 +202,14 @@ func (s *Scheduler) processPending(ctx context.Context) {if loadedCount == 0 {// No models loaded. Load the model but prefer the best fit.slog.Debug("loading first model", "model", pending.model.ModelPath)- s.loadFn(pending, ggml, gpus, false)+ s.loadFn(pending, ggml, systemInfo, gpus, false)break}// More than one loaded model, so we have to see if the// new one fits- needEvict := s.loadFn(pending, ggml, gpus, true)+ needEvict := s.loadFn(pending, ggml, systemInfo, gpus, true)if !needEvict {slog.Debug("new model fits with existing models, loading")break@@ -353,7 +340,7 @@ func (s *Scheduler) processCompleted(ctx context.Context) {runner.refMu.Unlock()} else {slog.Debug("starting background wait for VRAM recovery", "runner", runner)- runnersSnapshot := make([]discover.FilteredRunnerDiscovery, 0, len(s.loaded))+ runnersSnapshot := make([]ml.FilteredRunnerDiscovery, 0, len(s.loaded))for _, r := range s.loaded {runnersSnapshot = append(runnersSnapshot, r)}@@ -395,7 +382,7 @@ func (pending *LlmRequest) useLoadedRunner(runner *runnerRef, finished chan *Llm// load creates a new model based on req and loads it. If requireFull is true then the model must be loaded fully onto GPUs// (if any). Returns whether the scheduler needs to evict a model to make this one fit.-func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoList, requireFull bool) bool {+func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) bool {numParallel := max(int(envconfig.NumParallel()), 1)// Embedding models should always be loaded with parallel=1@@ -420,7 +407,7 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLisif llama == nil {var err error- llama, err = s.newServerFn(gpus, req.model.ModelPath, f, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, numParallel)+ llama, err = s.newServerFn(systemInfo, gpus, req.model.ModelPath, f, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, numParallel)if err != nil {// some older models are not compatible with newer versions of llama.cpp// show a generalized compatibility error until there is a better way to@@ -443,9 +430,16 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLiss.loadedMu.Unlock()- gpuIDs, err := llama.Load(req.ctx, gpus, requireFull)+ gpuIDs, err := llama.Load(req.ctx, systemInfo, gpus, requireFull)if err != nil {if errors.Is(err, llm.ErrLoadRequiredFull) {+ if !requireFull {+ // No other models loaded, yet we still don't fit, so report an error+ slog.Info("model is too large for system memory", "requireFull", requireFull)+ s.activeLoading.Close()+ s.activeLoading = nil+ req.errCh <- err+ }return true}@@ -456,6 +450,20 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLisreturn false}+ // Determine if we have discrete GPUs which we should monitor VRAM usage on during shutdown+ discreteGPUs := false+iGPUScan:+ for _, devid := range gpuIDs {+ for _, dev := range gpus {+ if dev.DeviceID == devid {+ if !dev.Integrated {+ discreteGPUs = true+ break iGPUScan+ }+ }+ }+ }+runner := &runnerRef{model: req.model,modelPath: req.model.ModelPath,@@ -463,6 +471,7 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLisOptions: &req.opts,sessionDuration: sessionDuration,gpus: gpuIDs,+ discreteGPUs: discreteGPUs,vramSize: llama.VRAMSize(),totalSize: llama.TotalSize(),loading: true,@@ -510,7 +519,10 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLisreturn false}-func (s *Scheduler) updateFreeSpace(allGpus discover.GpuInfoList) {+func (s *Scheduler) updateFreeSpace(allGpus []ml.DeviceInfo) {+ if len(allGpus) == 0 {+ return+ }predMap := map[ml.DeviceID]uint64{} // Sum up the total predicted usage per GPU for all runnerss.loadedMu.Lock()runners := make([]*runnerRef, 0, len(s.loaded))@@ -554,12 +566,13 @@ type runnerRef struct {refMu sync.MutexrefCount uint // prevent unloading if > 0- llama llm.LlamaServer- pid int- loading bool // True only during initial load, then false forever- gpus []ml.DeviceID // Recorded at time of provisioning- vramSize uint64- totalSize uint64+ llama llm.LlamaServer+ pid int+ loading bool // True only during initial load, then false forever+ gpus []ml.DeviceID // Recorded at time of provisioning+ discreteGPUs bool // True if all devices are discrete GPUs - used to skip VRAM recovery check for iGPUs+ vramSize uint64+ totalSize uint64sessionDuration time.DurationexpireTimer *time.Timer@@ -627,14 +640,12 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool// a before and after GPU memory allocation. The returned channel// will be notified when we're done waiting, or have timed out and should// proceed anyway-func (s *Scheduler) waitForVRAMRecovery(runner *runnerRef, runners []discover.FilteredRunnerDiscovery) chan any {+func (s *Scheduler) waitForVRAMRecovery(runner *runnerRef, runners []ml.FilteredRunnerDiscovery) chan any {finished := make(chan any, 1)- // CPU or Metal don't need checking, so no waiting required- // windows can page VRAM, only cuda currently can report accurate used vram usage- if len(runner.gpus) == 0 ||- (len(runner.gpus) == 1 && (runner.gpus[0].Library == "cpu" || runner.gpus[0].Library == "Metal")) ||- (runtime.GOOS == "windows" && runner.gpus[0].Library != "CUDA") {+ // CPU, Metal and iGPUs don't need checking, so no waiting required+ if len(runner.gpus) == 0 || !runner.discreteGPUs ||+ (len(runner.gpus) == 1 && runner.gpus[0].Library == "Metal") {finished <- struct{}{}slog.Debug("no need to wait for VRAM recovery", "runner", runner)return finished@@ -668,7 +679,11 @@ func (s *Scheduler) waitForVRAMRecovery(runner *runnerRef, runners []discover.FitotalMemoryNow += gpu.TotalMemoryfreeMemoryNow += gpu.FreeMemory}- logutil.Trace("gpu VRAM convergence", "percent", int(max(float32(freeMemoryNow-freeMemoryBefore), 0.0)/float32(runner.vramSize)*100))+ if freeMemoryNow > freeMemoryBefore {+ logutil.Trace("gpu VRAM convergence", "percent", int(float32(freeMemoryNow-freeMemoryBefore)/float32(runner.vramSize)*100))+ } else {+ logutil.Trace("gpu VRAM convergence", "percent", 0)+ }// If we're within ~75% of the estimated memory usage recovered, bail outif float32(freeMemoryNow-freeMemoryBefore) > float32(runner.vramSize)*0.75 {slog.Debug(fmt.Sprintf("gpu VRAM free memory converged after %0.2f seconds", time.Since(start).Seconds()), "free_before", format.HumanBytes2(freeMemoryBefore), "free_now", format.HumanBytes2(freeMemoryNow), "runner", runner)diff --git a/server/sched_test.go b/server/sched_test.goindex 66d43338e..316a817fe 100644--- a/server/sched_test.go+++ b/server/sched_test.go@@ -13,7 +13,6 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/app/lifecycle"- "github.com/ollama/ollama/discover""github.com/ollama/ollama/format""github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/llm"@@ -50,11 +49,12 @@ func TestSchedLoad(t *testing.T) {sessionDuration: &api.Duration{Duration: 2 * time.Second},}// Fail to load model first- s.newServerFn = func(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {return nil, errors.New("something failed to load model blah")}- gpus := discover.GpuInfoList{}- s.load(req, f, gpus, false)+ gpus := []ml.DeviceInfo{}+ systemInfo := ml.SystemInfo{}+ s.load(req, f, systemInfo, gpus, false)require.Empty(t, req.successCh)require.Len(t, req.errCh, 1)s.loadedMu.Lock()@@ -64,11 +64,11 @@ func TestSchedLoad(t *testing.T) {require.Contains(t, err.Error(), "this model may be incompatible")server := &mockLlm{vramSize: 10, vramByGPU: map[ml.DeviceID]uint64{}}- s.newServerFn = func(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {server.modelPath = modelreturn server, nil}- s.load(req, f, gpus, false)+ s.load(req, f, systemInfo, gpus, false)select {case err := <-req.errCh:require.NoError(t, err)@@ -82,7 +82,7 @@ func TestSchedLoad(t *testing.T) {req.model.ModelPath = "dummy_model_path"server.waitResp = errors.New("wait failure")- s.load(req, f, gpus, false)+ s.load(req, f, systemInfo, gpus, false)select {case err := <-req.errCh:require.Contains(t, err.Error(), "wait failure")@@ -106,7 +106,7 @@ type reqBundle struct {f *ggml.GGML}-func (scenario *reqBundle) newServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+func (scenario *reqBundle) newServer(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {scenario.srv.modelPath = modelreturn scenario.srv, nil}@@ -152,20 +152,20 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, vrareturn b}-func getGpuFn(ctx context.Context, runners []discover.FilteredRunnerDiscovery) discover.GpuInfoList {+func getGpuFn(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.DeviceInfo {slog.Info("test getGpuFn called", "runners", runners)- g := discover.GpuInfo{DeviceID: ml.DeviceID{Library: "metal"}}+ g := ml.DeviceInfo{DeviceID: ml.DeviceID{Library: "Metal"}}g.TotalMemory = 24 * format.GigaByteg.FreeMemory = 12 * format.GigaByte- return []discover.GpuInfo{g}+ return []ml.DeviceInfo{g}}-func getCpuFn() discover.GpuInfo {- slog.Info("test getCpuFn called")- g := discover.GpuInfo{DeviceID: ml.DeviceID{Library: "cpu"}}- g.TotalMemory = 32 * format.GigaByte- g.FreeMemory = 26 * format.GigaByte- return g+func getSystemInfoFn() ml.SystemInfo {+ slog.Info("test getSystemInfoFn called")+ return ml.SystemInfo{+ TotalMemory: 32 * format.GigaByte,+ FreeMemory: 26 * format.GigaByte,+ }}func TestSchedRequestsSameModelSameRequest(t *testing.T) {@@ -174,7 +174,7 @@ func TestSchedRequestsSameModelSameRequest(t *testing.T) {s := InitScheduler(ctx)s.waitForRecovery = 10 * time.Milliseconds.getGpuFn = getGpuFn- s.getCpuFn = getCpuFn+ s.getSystemInfoFn = getSystemInfoFna := newScenarioRequest(t, ctx, "ollama-model-1", 10, &api.Duration{Duration: 5 * time.Millisecond}, nil)b := newScenarioRequest(t, ctx, "ollama-model-1", 11, &api.Duration{Duration: 0}, nil)b.req.model = a.req.model@@ -218,7 +218,7 @@ func TestSchedRequestsSimpleReloadSameModel(t *testing.T) {s := InitScheduler(ctx)s.waitForRecovery = 10 * time.Milliseconds.getGpuFn = getGpuFn- s.getCpuFn = getCpuFn+ s.getSystemInfoFn = getSystemInfoFna := newScenarioRequest(t, ctx, "ollama-model-1", 10, &api.Duration{Duration: 5 * time.Millisecond}, nil)b := newScenarioRequest(t, ctx, "ollama-model-1", 20, &api.Duration{Duration: 5 * time.Millisecond}, nil)tmpModel := *a.req.model@@ -251,12 +251,12 @@ func TestSchedRequestsSimpleReloadSameModel(t *testing.T) {a.ctxDone()// Report recovered VRAM usagetime.Sleep(1 * time.Millisecond)- s.getGpuFn = func(ctx context.Context, runners []discover.FilteredRunnerDiscovery) discover.GpuInfoList {- slog.Info("XXX altered getGpuFn called")- g := discover.GpuInfo{DeviceID: ml.DeviceID{Library: "metal"}}+ s.getGpuFn = func(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.DeviceInfo {+ slog.Info("altered getGpuFn called")+ g := ml.DeviceInfo{DeviceID: ml.DeviceID{Library: "Metal"}}g.TotalMemory = 24 * format.GigaByteg.FreeMemory = 24 * format.GigaByte- return []discover.GpuInfo{g}+ return []ml.DeviceInfo{g}}select {case resp := <-b.req.successCh:@@ -271,26 +271,26 @@ func TestSchedRequestsSimpleReloadSameModel(t *testing.T) {}func TestSchedRequestsMultipleLoadedModels(t *testing.T) {- ctx, done := context.WithTimeout(t.Context(), 500*time.Millisecond)+ slog.Info("TestRequestsMultipleLoadedModels")+ ctx, done := context.WithTimeout(t.Context(), 1000*time.Millisecond)defer done()s := InitScheduler(ctx)s.waitForRecovery = 10 * time.Millisecond- s.getGpuFn = getGpuFn // 1 metal GPU- s.getCpuFn = getCpuFn // 1 CPU+ s.getGpuFn = getGpuFn // 1 Metal GPU+ s.getSystemInfoFn = getSystemInfoFn// Multiple loaded models- a := newScenarioRequest(t, ctx, "model-a-1g-gpu", 1*format.GigaByte, nil, map[ml.DeviceID]uint64{{Library: "metal"}: 1 * format.GigaByte})+ a := newScenarioRequest(t, ctx, "model-a-1g-gpu", 1*format.GigaByte, nil, map[ml.DeviceID]uint64{{Library: "Metal"}: 1 * format.GigaByte})a.req.sessionDuration = &api.Duration{Duration: 5 * time.Millisecond}- b := newScenarioRequest(t, ctx, "model-b-10g-gpu", 10*format.GigaByte, nil, map[ml.DeviceID]uint64{{Library: "metal"}: 10 * format.GigaByte})+ b := newScenarioRequest(t, ctx, "model-b-10g-gpu", 10*format.GigaByte, nil, map[ml.DeviceID]uint64{{Library: "Metal"}: 10 * format.GigaByte})b.req.sessionDuration = &api.Duration{Duration: 5 * time.Millisecond}c := newScenarioRequest(t, ctx, "model-c-10g-cpu", 10*format.GigaByte, nil, nil /* No GPU load */)c.req.opts.NumGPU = 0 // CPU load, will be allowedb.req.sessionDuration = &api.Duration{Duration: 10 * time.Millisecond} // longer than b to cause the scheduler to favor unloading b over c- d := newScenarioRequest(t, ctx, "model-d-10g-gpu", 13*format.GigaByte, nil, map[ml.DeviceID]uint64{{Library: "metal"}: 13 * format.GigaByte}) // Needs prior unloaded+ d := newScenarioRequest(t, ctx, "model-d-10g-gpu", 13*format.GigaByte, nil, map[ml.DeviceID]uint64{{Library: "Metal"}: 13 * format.GigaByte}) // Needs prior unloaded- t.Setenv("OLLAMA_MAX_LOADED_MODELS", "1")s.newServerFn = a.newServer- slog.Info("a")+ slog.Info("Loading A")s.pendingReqCh <- a.reqs.Run(ctx)select {@@ -309,7 +309,7 @@ func TestSchedRequestsMultipleLoadedModels(t *testing.T) {t.Setenv("OLLAMA_MAX_LOADED_MODELS", "0")s.newServerFn = b.newServer- slog.Info("b")+ slog.Info("Loading B")s.pendingReqCh <- b.reqselect {case resp := <-b.req.successCh:@@ -327,7 +327,7 @@ func TestSchedRequestsMultipleLoadedModels(t *testing.T) {// This is a CPU load with NumGPU = 0 so it should loads.newServerFn = c.newServer- slog.Info("c")+ slog.Info("Loading C")s.pendingReqCh <- c.reqselect {case resp := <-c.req.successCh:@@ -337,6 +337,7 @@ func TestSchedRequestsMultipleLoadedModels(t *testing.T) {case err := <-c.req.errCh:t.Fatal(err.Error())case <-ctx.Done():+ slog.Info("FAIL: scheduler state", "s.loaded", s.loaded)t.Fatal("timeout")}s.loadedMu.Lock()@@ -361,11 +362,11 @@ func TestSchedRequestsMultipleLoadedModels(t *testing.T) {b.ctxDone()// Report recovered VRAM usage so scheduler will finish waiting and unloadtime.Sleep(1 * time.Millisecond)- s.getGpuFn = func(ctx context.Context, runners []discover.FilteredRunnerDiscovery) discover.GpuInfoList {- g := discover.GpuInfo{DeviceID: ml.DeviceID{Library: "metal"}}+ s.getGpuFn = func(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.DeviceInfo {+ g := ml.DeviceInfo{DeviceID: ml.DeviceID{Library: "Metal"}}g.TotalMemory = 24 * format.GigaByteg.FreeMemory = 24 * format.GigaByte- return []discover.GpuInfo{g}+ return []ml.DeviceInfo{g}}select {case resp := <-d.req.successCh:@@ -404,7 +405,7 @@ func TestSchedGetRunner(t *testing.T) {s := InitScheduler(ctx)s.waitForRecovery = 10 * time.Milliseconds.getGpuFn = getGpuFn- s.getCpuFn = getCpuFn+ s.getSystemInfoFn = getSystemInfoFns.newServerFn = a.newServerslog.Info("a")successCh1a, errCh1a := s.GetRunner(a.ctx, a.req.model, a.req.opts, a.req.sessionDuration)@@ -462,13 +463,14 @@ func TestSchedExpireRunner(t *testing.T) {}var f *ggml.GGML- gpus := discover.GpuInfoList{}+ gpus := []ml.DeviceInfo{}+ systemInfo := ml.SystemInfo{}server := &mockLlm{vramSize: 10, vramByGPU: map[ml.DeviceID]uint64{}}- s.newServerFn = func(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {+ s.newServerFn = func(systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, model string, f *ggml.GGML, adapters []string, projectors []string, opts api.Options, numParallel int) (llm.LlamaServer, error) {server.modelPath = modelreturn server, nil}- s.load(req, f, gpus, false)+ s.load(req, f, systemInfo, gpus, false)select {case err := <-req.errCh:@@ -497,19 +499,15 @@ func TestSchedExpireRunner(t *testing.T) {// TODO - add one scenario that triggers the bogus finished event with positive ref countfunc TestSchedPrematureExpired(t *testing.T) {- ctx, done := context.WithTimeout(t.Context(), 500*time.Millisecond)+ ctx, done := context.WithTimeout(t.Context(), 1000*time.Millisecond)defer done()// Same model, same request- scenario1a := newScenarioRequest(t, ctx, "ollama-model-1a", 10, nil, nil)+ scenario1a := newScenarioRequest(t, ctx, "ollama-model-1a", 10, &api.Duration{Duration: 100 * time.Millisecond}, nil)s := InitScheduler(ctx)s.waitForRecovery = 10 * time.Millisecond- s.getGpuFn = func(ctx context.Context, runners []discover.FilteredRunnerDiscovery) discover.GpuInfoList {- g := discover.GpuInfo{DeviceID: ml.DeviceID{Library: "metal"}}- g.TotalMemory = 24 * format.GigaByte- g.FreeMemory = 12 * format.GigaByte- return []discover.GpuInfo{g}- }+ s.getGpuFn = getGpuFn+ s.getSystemInfoFn = getSystemInfoFns.newServerFn = scenario1a.newServersuccessCh1a, errCh1a := s.GetRunner(scenario1a.ctx, scenario1a.req.model, scenario1a.req.opts, scenario1a.req.sessionDuration)require.Len(t, s.pendingReqCh, 1)@@ -574,7 +572,7 @@ func TestSchedUseLoadedRunner(t *testing.T) {func TestSchedUpdateFreeSpace(t *testing.T) {ctx, done := context.WithTimeout(t.Context(), 100*time.Millisecond)defer done()- gpus := discover.GpuInfoList{+ gpus := []ml.DeviceInfo{{DeviceID: ml.DeviceID{ID: "1",@@ -756,8 +754,12 @@ func (s *mockLlm) ModelPath() string {return s.modelPath}-func (s *mockLlm) Load(ctx context.Context, gpus discover.GpuInfoList, requireFull bool) ([]ml.DeviceID, error) {+func (s *mockLlm) Load(ctx context.Context, sytemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {if requireFull {+ if len(gpus) == 0 {+ slog.Info("mockLlm.Load CPU based load")+ return nil, nil+ }for _, g := range gpus {if g.FreeMemory >= s.vramSize {return []ml.DeviceID{g.DeviceID}, nil