Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Clean up dead code (#17381)
Daniel Hiltgen68793119dffb
diff --git a/discover/llama_server.go b/discover/llama_server.goindex a66c34e87..ba8ace58c 100644--- a/discover/llama_server.go+++ b/discover/llama_server.go@@ -199,13 +199,9 @@ var (cudaRuntimeDirRegex = regexp.MustCompile(`^cuda_v(\d+)$`))-// parseLlamaServerDevices parses the combined output of llama-server discovery.-// It extracts device info, ROCm gfx targets, CUDA compute capabilities, and-// CUDA compiled architecture lists.-func parseLlamaServerDevices(output string, libDirs []string) []ml.DeviceInfo {- return parseLlamaServerDevicesWithNative(output, "", libDirs, nil)-}-+// parseLlamaServerDevicesWithNative parses the combined output of llama-server+// discovery. It extracts device info, ROCm gfx targets, CUDA compute+// capabilities, and CUDA compiled architecture lists.func parseLlamaServerDevicesWithNative(output, nativeOutput string, libDirs []string, nativeDevices []nativeProbeDevice) []ml.DeviceInfo {combined := outputif nativeOutput != "" {diff --git a/discover/llama_server_test.go b/discover/llama_server_test.goindex 563da94e1..0418dc2fb 100644--- a/discover/llama_server_test.go+++ b/discover/llama_server_test.go@@ -243,7 +243,7 @@ Available devices:if tt.libDirs == nil {tt.libDirs = []string{"/lib/ollama"}}- devices := parseLlamaServerDevices(tt.output, tt.libDirs)+ devices := parseLlamaServerDevicesWithNative(tt.output, "", tt.libDirs, nil)if len(devices) != len(tt.want) {t.Fatalf("got %d devices, want %d", len(devices), len(tt.want))}@@ -357,7 +357,7 @@ Available devices:libDirs = []string{"/lib/ollama"}}- got := parseLlamaServerDevices(tt.output, libDirs)+ got := parseLlamaServerDevicesWithNative(tt.output, "", libDirs, nil)if len(got) != len(tt.want) {t.Fatalf("got %d devices, want %d", len(got), len(tt.want))}diff --git a/ml/backend.go b/ml/backend.goindex 37efd365e..7c3340edd 100644--- a/ml/backend.go+++ b/ml/backend.go@@ -19,9 +19,6 @@ type Backend interface {Load(ctx context.Context, progress func(float32)) error- // BackendMemory returns the memory allocations that were made for this model- BackendMemory() BackendMemory-Config() fs.ConfigGet(name string) TensorNewContext() Context@@ -66,9 +63,6 @@ type BackendParams struct {// NumThreads sets the number of threads to use if running on the CPUNumThreads int- // GPULayers is the set of layers to offload to GPUs- GPULayers GPULayersList-// FlashAttention indicates that we should use a fused flash attention kernelFlashAttention FlashAttentionType}diff --git a/ml/device.go b/ml/device.goindex fbeb07139..408f1cd7d 100644--- a/ml/device.go+++ b/ml/device.go@@ -2,133 +2,17 @@ package mlimport ("context"- "encoding/binary"- "encoding/json""fmt"- "hash/maphash"- "io""log/slog"- "math"- "net/http""os""runtime"- "slices""sort""strconv""strings"- "time""github.com/ollama/ollama/format"- "github.com/ollama/ollama/logutil")-// GPULayers is a set of layers to be allocated on a single GPU-type GPULayers struct {- DeviceID-- // Layers is a set of layer indicies to load- Layers []int-}--// FirstLayer returns the smallest layer index scheduled on this GPU, or MaxInt when empty.-func (g GPULayers) FirstLayer() int {- if len(g.Layers) == 0 {- return math.MaxInt- }-- first := g.Layers[0]- for i := 1; i < len(g.Layers); i++ {- if g.Layers[i] < first {- first = g.Layers[i]- }- }-- return first-}--func (g GPULayers) String() string {- if len(g.Layers) == 0 {- return ""- }-- slices.Sort(g.Layers)-- contiguous := true- base := g.Layers[0]- for i := range g.Layers {- if g.Layers[i] != base+i {- contiguous = false- break- }- }-- if contiguous {- return fmt.Sprintf("ID:%v Layers:%v(%v..%v)", g.ID, len(g.Layers), g.Layers[0], g.Layers[len(g.Layers)-1])- } else {- return fmt.Sprintf("ID:%v Layers:%v%v", g.ID, len(g.Layers), g.Layers)- }-}--// GPULayersList is a set of layer allocations across multiple GPUs-type GPULayersList []GPULayers--func (l GPULayersList) Len() int { return len(l) }-func (l GPULayersList) Swap(i, j int) { l[i], l[j] = l[j], l[i] }--// Sort by the ordering of the layers offloaded-func (l GPULayersList) Less(i, j int) bool {- li := l[i].FirstLayer()- lj := l[j].FirstLayer()-- return li < lj-}--func (l GPULayersList) String() string {- if l.Sum() > 0 {- return fmt.Sprintf("%v%v", l.Sum(), []GPULayers(l))- } else {- return fmt.Sprintf("%v", []GPULayers(l))- }-}--// Sum is the total number of layers assigned across all GPUs-func (l GPULayersList) Sum() int {- var sum int-- for _, g := range l {- sum += len(g.Layers)- }-- return sum-}--var h maphash.Hash--// Hash is an identifier of this layer assignment-func (l GPULayersList) Hash() uint64 {- h.Reset()- for _, g := range l {- if len(g.Layers) > 0 {- h.WriteString(g.ID + g.Library)- for _, l := range g.Layers {- binary.Write(&h, binary.NativeEndian, int64(l))- }- }- }-- return h.Sum64()-}--// ErrNoMem is returned when panicing due to insufficient memory. It includes-// the attempted memory allocation.-type ErrNoMem struct {- BackendMemory-}--func (e ErrNoMem) Error() string {- return fmt.Sprintf("insufficient memory - required allocations: %+v", e.BackendMemory)-}-// Minimal unique device identificationtype DeviceID struct {// ID is an identifier for the device for matching with system@@ -142,138 +26,6 @@ type DeviceID struct {Library string `json:"backend,omitempty"`}-// DeviceMemory provides a breakdown of the memory needed-// per device, such as a CPU or GPU.-type DeviceMemory struct {- DeviceID-- // Name is the name of the device as labeled by the backend. It- // may not be persistent across instances of the runner.- Name string-- // Weights is the per-layer memory needed for the model weights.- Weights []uint64-- // Cache is the per-layer memory needed for the KV cache.- Cache []uint64-- // Graph is the size of the compute graph. It is not per-layer.- Graph uint64-}--func sumMemory(mem []uint64) uint64 {- var sum uint64-- for _, m := range mem {- sum += m- }-- return sum-}--// Size returns the total size of the memory required by this device-func (m DeviceMemory) Size() uint64 {- return sumMemory(m.Weights) + sumMemory(m.Cache) + m.Graph-}--func memoryPresent(mem []uint64) bool {- return slices.ContainsFunc(mem, func(m uint64) bool { return m != 0 })-}--func (m DeviceMemory) LogValue() slog.Value {- var attrs []slog.Attr- if memoryPresent(m.Weights) {- attrs = append(attrs, slog.Any("Weights", m.Weights))- }-- if memoryPresent(m.Cache) {- attrs = append(attrs, slog.Any("Cache", m.Cache))- }-- if m.Graph != 0 {- attrs = append(attrs, slog.Any("Graph", m.Graph))- }-- if len(attrs) > 0 && m.ID != "" {- attrs = append([]slog.Attr{slog.String("ID", m.ID)}, attrs...)- }-- return slog.GroupValue(attrs...)-}--// BackendMemory provides the amount of memory required to load the model-// per device based on the BackendParams. In some cases, not all required-// allocations will be known at this point. However, the size of the most recent-// allocation is guaranteed to be provided so that if it failed, the caller can-// accommodate that to make forward progress.-type BackendMemory struct {- // InputWeights are always located on the CPU and cannot be moved- InputWeights uint64-- // CPU model components are located in system memory. This does not- // include unified memory allocated through the GPU.- CPU DeviceMemory-- // GPU model components are located on one or more GPUs.- GPUs []DeviceMemory-}--func (m BackendMemory) LogValue() slog.Value {- var attrs []slog.Attr- if m.InputWeights != 0 {- attrs = append(attrs, slog.Any("InputWeights", m.InputWeights))- }-- attrs = append(attrs, slog.Any(m.CPU.Name, m.CPU))- for _, g := range m.GPUs {- attrs = append(attrs, slog.Any(g.Name, g))- }-- return slog.GroupValue(attrs...)-}--// Log prints a high level summary of the memory-func (m BackendMemory) Log(level slog.Level) {- var total uint64-- for _, gpu := range m.GPUs {- if sum := sumMemory(gpu.Weights); sum > 0 {- slog.Log(context.TODO(), level, "model weights", "device", gpu.Name, "size", format.HumanBytes2(sum))- total += sum- }- }- if sum := m.InputWeights + sumMemory(m.CPU.Weights); sum > 0 {- slog.Log(context.TODO(), level, "model weights", "device", m.CPU.Name, "size", format.HumanBytes2(sum))- total += sum- }-- for _, gpu := range m.GPUs {- if sum := sumMemory(gpu.Cache); sum > 0 {- slog.Log(context.TODO(), level, "kv cache", "device", gpu.Name, "size", format.HumanBytes2(sum))- total += sum- }- }- if sum := sumMemory(m.CPU.Cache); sum > 0 {- slog.Log(context.TODO(), level, "kv cache", "device", m.CPU.Name, "size", format.HumanBytes2(sum))- total += sum- }-- for _, gpu := range m.GPUs {- if sum := gpu.Graph; sum > 0 {- slog.Log(context.TODO(), level, "compute graph", "device", gpu.Name, "size", format.HumanBytes2(sum))- total += sum- }- }- if sum := m.CPU.Graph; sum > 0 {- slog.Log(context.TODO(), level, "compute graph", "device", m.CPU.Name, "size", format.HumanBytes2(sum))- total += sum- }-- if total > 0 {- slog.Log(context.TODO(), level, "total memory", "size", format.HumanBytes2(total))- }-}-type DeviceInfo struct {DeviceID@@ -378,28 +130,6 @@ func (a ByFreeMemory) Less(i, j int) bool {return a[i].FreeMemory < a[j].FreeMemory}-// ByPerformance groups devices by similar speed-func ByPerformance(l []DeviceInfo) [][]DeviceInfo {- resp := [][]DeviceInfo{}- scores := []bool{}- for _, info := range l {- found := false- requested := info.Integrated- for i, score := range scores {- if score == requested {- resp[i] = append(resp[i], info)- found = true- break- }- }- if !found {- scores = append(scores, requested)- resp = append(resp, []DeviceInfo{info})- }- }- return resp-}-func ByLibrary(l []DeviceInfo) [][]DeviceInfo {resp := [][]DeviceInfo{}libs := []string{}@@ -810,51 +540,3 @@ type FilteredRunnerDiscovery interface {GetActiveDeviceIDs() []DeviceID}-func GetDevicesFromRunner(ctx context.Context, runner BaseRunner) ([]DeviceInfo, error) {- var moreDevices []DeviceInfo- port := runner.GetPort()- tick := time.Tick(10 * time.Millisecond)- for {- select {- case <-ctx.Done():- return nil, fmt.Errorf("failed to finish discovery before timeout")- case <-tick:- r, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/info", port), nil)- if err != nil {- return nil, fmt.Errorf("failed to create request: %w", err)- }- r.Header.Set("Content-Type", "application/json")-- resp, err := http.DefaultClient.Do(r)- if err != nil {- // slog.Warn("failed to send request", "error", err)- if runner.HasExited() {- return nil, fmt.Errorf("runner crashed")- }- continue- }- defer resp.Body.Close()-- if resp.StatusCode == http.StatusNotFound {- // old runner, fall back to bootstrapping model- return nil, fmt.Errorf("llamarunner free vram reporting not supported")- }-- body, err := io.ReadAll(resp.Body)- if err != nil {- slog.Warn("failed to read response", "error", err)- continue- }- if resp.StatusCode != 200 {- logutil.Trace("runner failed to discover free VRAM", "status", resp.StatusCode, "response", body)- return nil, fmt.Errorf("runner error: %s", string(body))- }-- if err := json.Unmarshal(body, &moreDevices); err != nil {- slog.Warn("unmarshal encode response", "error", err)- continue- }- return moreDevices, nil- }- }-}