Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llm: Use Ollama engine memory layouts for both old and new engines
Jesse Grossf560bd077fcf
diff --git a/fs/ggml/ggml.go b/fs/ggml/ggml.goindex 909104c64..0b5d37a7b 100644--- a/fs/ggml/ggml.go+++ b/fs/ggml/ggml.go@@ -797,73 +797,6 @@ func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType strireturn}-func (llm GGML) VisionGraphSize() (weights, graphSize uint64) {- if llm.KV().Uint("vision.block_count") == 0 {- return- }-- for name, layer := range llm.Tensors().GroupLayers() {- if name == "v" || strings.HasPrefix(name, "v.") {- for _, tensor := range layer {- weights += tensor.Size()- }- }- }-- imageSize := uint64(llm.KV().Uint("vision.image_size"))- patchSize := uint64(llm.KV().Uint("vision.patch_size"))- if patchSize == 0 {- slog.Warn("unknown patch size for vision model")- return- }-- numChannels := uint64(llm.KV().Uint("vision.num_channels"))-- numPatches := (imageSize / patchSize) * (imageSize / patchSize)- if _, ok := llm.Tensors().GroupLayers()["v"]["class_embd"]; ok {- numPatches++- }-- headCount := uint64(llm.KV().Uint("vision.attention.head_count"))- embeddingLength := uint64(llm.KV().Uint("vision.embedding_length"))-- switch llm.KV().Architecture() {- case "mllama":- numPaddedPatches := numPatches + 8 - (numPatches%8)%8-- maxNumTiles := uint64(llm.KV().Uint("vision.max_num_tiles"))-- graphSize = 4 * (8 +- imageSize*imageSize*numChannels*maxNumTiles +- embeddingLength*numPatches*maxNumTiles +- 9*embeddingLength*numPaddedPatches*maxNumTiles +- numPaddedPatches*maxNumTiles*numPaddedPatches*maxNumTiles*headCount)- case "gemma3", "mistral3":- graphSize = 4 * (imageSize*imageSize*numChannels +- embeddingLength*patchSize +- numPatches*numPatches*headCount)- case "qwen25vl":- maxPixels := uint64(llm.KV().Uint("vision.max_pixels", 28*28*1280))-- numPatches := maxPixels / (patchSize * patchSize)-- graphSize = 4 * (maxPixels*numChannels + // Original image storage- // Normalized pixels- maxPixels*numChannels +- // Patches storage (numPatches * channels * patchSize^2)- numPatches*numChannels*patchSize*patchSize +- // Self-attention calculations- numPatches*numPatches*headCount +- // Additional buffer for processing- embeddingLength*numPatches)- case "llama4":- // vision graph is computed independently in the same schedule- // and is negligible compared to the worst case text graph- }-- return weights, graphSize-}-// SupportsKVCacheType checks if the requested cache type is supportedfunc (f GGML) SupportsKVCacheType(cacheType string) bool {if cacheType == "" || cacheType == "f16" {diff --git a/llm/memory.go b/llm/memory.godeleted file mode 100644index 15558109f..000000000--- a/llm/memory.go+++ /dev/null@@ -1,516 +0,0 @@-package llm--import (- "fmt"- "log/slog"- "os"- "slices"- "sort"- "strings"-- "github.com/ollama/ollama/api"- "github.com/ollama/ollama/envconfig"- "github.com/ollama/ollama/format"- "github.com/ollama/ollama/fs/ggml"- "github.com/ollama/ollama/ml"-)--// pickBestFullFitByLibrary will try to find the optimal placement of the model in the available GPUs where the model fully fits-// The list of GPUs returned will always be the same brand (library)-// If the model can not be fit fully within the available GPU(s) nil is returned-func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, gpus []ml.DeviceInfo, numParallel int) []ml.DeviceInfo {- for _, gl := range ml.ByLibrary(gpus) {- sgl := append(make([]ml.DeviceInfo, 0, len(gl)), gl...)-- // TODO - potentially sort by performance capability, existing models loaded, etc.- // TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them- // Note: at present, this will favor most current available VRAM descending and ignoring faster GPU speed in mixed setups- sort.Sort(sort.Reverse(ml.ByFreeMemory(sgl)))-- if !envconfig.SchedSpread() {- // Try to pack into as few GPUs as possible, starting from 1 GPU- for numGPUs := 1; numGPUs <= len(sgl); numGPUs++ {- gpuSubset := sgl[:numGPUs]- ok, estimatedVRAM := predictServerFit(gpuSubset, f, adapters, projectors, opts, numParallel)-- if ok {- slog.Info("new model will fit in available VRAM across minimum required GPUs, loading",- "model", modelPath,- "library", sgl[0].Library,- "parallel", numParallel,- "required", format.HumanBytes2(estimatedVRAM),- "gpus", numGPUs)- return gpuSubset- }- }- } else {- // TODO future refinements- // - if multiple Libraries, see if any single GPU in any Library will fit- // - try subsets of GPUs instead of just falling back to 1 or all in a family-- // Now try all the GPUS (OLLAMA_SCHED_SPREAD is set)- if ok, estimatedVRAM := predictServerFit(sgl, f, adapters, projectors, opts, numParallel); ok {- slog.Info("new model will fit in available VRAM, loading",- "model", modelPath,- "library", sgl[0].Library,- "parallel", numParallel,- "required", format.HumanBytes2(estimatedVRAM),- "gpus", len(sgl))- return sgl- }- }- }- return nil-}--// If multiple Libraries are detected, pick the Library which loads the most layers for the model-func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []string, opts api.Options, gpus []ml.DeviceInfo, numParallel int) []ml.DeviceInfo {- byLibrary := ml.ByLibrary(gpus)- if len(byLibrary) <= 1 {- return gpus- }- var bestEstimate uint64- var bestFit int- for i, gl := range byLibrary {- _, estimatedVRAM := predictServerFit(gl, f, adapters, projectors, opts, numParallel)- if estimatedVRAM > bestEstimate {- bestEstimate = estimatedVRAM- bestFit = i- }- }- return byLibrary[bestFit]-}--// This algorithm looks for a complete fit to determine if we need to unload other models-func predictServerFit(allGpus []ml.DeviceInfo, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64) {- // Split up the GPUs by type and try them- var estimatedVRAM uint64- for _, gpus := range ml.ByLibrary(allGpus) {- var layerCount int- estimate := estimateGPULayers(gpus, f, projectors, opts, numParallel)- layerCount, estimatedVRAM = estimate.Layers, estimate.VRAMSize- if opts.NumGPU < 0 {- if layerCount > 0 && layerCount >= int(f.KV().BlockCount()+1) {- return true, estimatedVRAM- }- } else {- if layerCount > 0 && layerCount >= opts.NumGPU {- return true, estimatedVRAM- }- }- }- return false, estimatedVRAM-}--func verifyCPUFit(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, systemInfo ml.SystemInfo, numParallel int) bool {- estimate := estimateGPULayers(nil, f, projectors, opts, numParallel)- if estimate.TotalSize > systemInfo.FreeMemory {- return false- }- slog.Info("new model will fit in available system memory for CPU inference, loading",- "model", modelPath,- "parallel", numParallel,- "required", format.HumanBytes2(estimate.TotalSize),- )- return true-}--type MemoryEstimate struct {- // How many layers we predict we can load- Layers int-- // The size of the graph which occupies the main GPU- Graph uint64-- // How much VRAM will be allocated given the number of layers we predict- VRAMSize uint64-- // The total size of the model if loaded into VRAM. If all layers are loaded, VRAMSize == TotalSize- TotalSize uint64-- // For multi-GPU scenarios, this provides the tensor split parameter- TensorSplit []int-- // For multi-GPU scenarios, this is the size in bytes per GPU- GPUSizes []uint64-- // internal fields for logging purposes- inferenceLibrary string- layersRequested int- layersModel int- availableList []string- kv uint64- allocationsList []string- memoryWeights uint64- memoryLayerOutput uint64- graphFullOffload uint64- graphPartialOffload uint64-- projectorWeights, projectorGraph uint64-}--// Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size-// The GPUs provided must all be the same Library-func estimateGPULayers(gpus []ml.DeviceInfo, f *ggml.GGML, projectors []string, opts api.Options, numParallel int) MemoryEstimate {- // Graph size for a partial offload, applies to all GPUs- var graphPartialOffload uint64-- // Graph size when all layers are offloaded, applies to all GPUs- var graphFullOffload uint64-- // Final graph offload once we know full or partial- var graphOffload uint64-- // Projectors loaded into GPU0 only- var llamaEngineProjectorWeights uint64-- // Projectors loaded with output layer- var ollamaEngineProjectorWeights uint64- var ollamaEngineProjectorGraph uint64-- // Conditional output size on GPU 0- var memoryLayerOutput uint64-- // The sizes of a layer- var layerSize uint64-- // The sum of all the layer sizes (just for logging)- var memoryWeights uint64-- // True if all the layers are loaded- var fullyLoaded bool-- // Overflow that didn't fit into the GPU- var overflow uint64-- overhead := envconfig.GpuOverhead()- availableList := make([]string, len(gpus))- libraries := []string{}- for i, gpu := range gpus {- availableList[i] = format.HumanBytes2(gpu.FreeMemory)- if !slices.Contains(libraries, gpu.Library) {- libraries = append(libraries, gpu.Library)- }- }- if len(libraries) == 0 {- libraries = []string{"cpu"}- }- slog.Debug("evaluating", "library", strings.Join(libraries, ","), "gpu_count", len(gpus), "available", availableList)-- for _, projector := range projectors {- llamaEngineProjectorWeights += projectorMemoryRequirements(projector)- }- if llamaEngineProjectorWeights == 0 {- ollamaEngineProjectorWeights, ollamaEngineProjectorGraph = f.VisionGraphSize()- }-- layers := f.Tensors().GroupLayers()- // add one layer worth of memory as a buffer- if blk0, ok := layers["blk.0"]; ok {- layerSize = blk0.Size()- } else {- slog.Warn("model missing blk.0 layer size")- }-- useFlashAttention := envconfig.FlashAttention(f.FlashAttention()) &&- ml.FlashAttentionSupported(gpus) &&- f.SupportsFlashAttention()-- var kvct string- if useFlashAttention {- requested := strings.ToLower(envconfig.KvCacheType())- if f.SupportsKVCacheType(requested) {- kvct = requested- }- }-- kv, graphPartialOffload, graphFullOffload := f.GraphSize(uint64(opts.NumCtx), uint64(min(opts.NumCtx, opts.NumBatch)), numParallel, kvct, useFlashAttention)-- if len(kv) > 0 {- layerSize += kv[0]- }-- var kvTotal uint64- for _, kvLayer := range kv {- kvTotal += kvLayer- }-- if graphPartialOffload == 0 {- headsKV := f.KV().HeadCountKVMin()- if headsKV == 0 {- headsKV = 1- }- gqa := f.KV().HeadCountMax() / headsKV- graphPartialOffload = gqa * kvTotal / 6- }- if graphFullOffload == 0 {- graphFullOffload = graphPartialOffload- }-- // on metal there's no partial offload overhead- if len(gpus) > 0 && gpus[0].Library == "Metal" {- graphPartialOffload = graphFullOffload- } else if len(gpus) > 1 {- // multigpu should always use the partial graph size- graphFullOffload = graphPartialOffload- }-- // Output layer handled at the end if we have space- if layer, ok := layers["output_norm"]; ok {- memoryLayerOutput += layer.Size()- }- if layer, ok := layers["output"]; ok {- memoryLayerOutput += layer.Size()- } else if layer, ok := layers["token_embd"]; ok {- memoryLayerOutput += layer.Size()- }-- gpuZeroOverhead := llamaEngineProjectorWeights-- // Reduce set of GPUs to only those that have sufficient space to fit overhead and at least one layer- var layerCount int- tensorSplit := make([]int, len(gpus))- gpuAllocations := make([]uint64, len(gpus))- type gs struct {- i int- g *ml.DeviceInfo- }- gpusWithSpace := []gs{}- for i := range gpus {- var gzo uint64- if len(gpusWithSpace) == 0 {- gzo = gpuZeroOverhead- }- // Only include GPUs that can fit the graph, gpu minimum, the layer buffer and at least more layer- if gpus[i].FreeMemory < overhead+gzo+max(graphPartialOffload, graphFullOffload)+gpus[i].MinimumMemory()+2*layerSize {- slog.Debug("gpu has too little memory to allocate any layers",- "id", gpus[i].ID,- "library", gpus[i].Library,- "compute", gpus[i].Compute(),- "driver", fmt.Sprintf("%d.%d", gpus[i].DriverMajor, gpus[i].DriverMinor),- "name", gpus[i].Name,- "total", format.HumanBytes2(gpus[i].TotalMemory),- "available", format.HumanBytes2(gpus[i].FreeMemory),- "minimum_memory", gpus[i].MinimumMemory,- "layer_size", format.HumanBytes2(layerSize),- "gpu_zer_overhead", format.HumanBytes2(gzo),- "partial_offload", format.HumanBytes2(graphPartialOffload),- "full_offload", format.HumanBytes2(graphFullOffload),- )- continue- }- gpusWithSpace = append(gpusWithSpace, gs{i, &gpus[i]})- gpuAllocations[i] += gpus[i].MinimumMemory() + layerSize // We hold off on graph until we know partial vs. full- }-- var gpuZeroID int- if len(gpusWithSpace) > 0 {- gpuZeroID = gpusWithSpace[0].i- gpuAllocations[gpuZeroID] += gpuZeroOverhead- } else {- overflow += gpuZeroOverhead- }-- // For all the layers, find where they can fit on the GPU(s)- for i := int(f.KV().BlockCount()) - 1; i >= 0; i-- {- // Some models have inconsistent layer sizes- if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {- layerSize = blk.Size()- layerSize += kv[i]- memoryWeights += blk.Size()- }-- if opts.NumGPU >= 0 && layerCount >= opts.NumGPU {- // Stop allocating on GPU(s) once we hit the users target NumGPU- overflow += layerSize- continue- }-- // distribute the layers across the GPU(s) that have space- for j := len(gpusWithSpace); j > 0; j-- {- g := gpusWithSpace[i%j]- used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)- if g.g.FreeMemory > overhead+used+layerSize {- gpuAllocations[g.i] += layerSize- tensorSplit[g.i]++- layerCount++- break- } else {- gpusWithSpace = append(gpusWithSpace[:i%j], gpusWithSpace[i%j+1:]...)- }- }-- if len(gpusWithSpace) == 0 {- overflow += layerSize- }- }- if layerCount >= int(f.KV().BlockCount()) {- fullyLoaded = true- }-- // Determine if we need to consider output then find where it fits- memoryLastLayer := memoryLayerOutput + ollamaEngineProjectorWeights + ollamaEngineProjectorGraph- if memoryLastLayer > 0 {- if opts.NumGPU < 0 || layerCount < opts.NumGPU {- for j := len(gpusWithSpace); j > 0; j-- {- g := gpusWithSpace[layerCount%j]- used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)- if g.g.FreeMemory > overhead+used+memoryLastLayer {- gpuAllocations[g.i] += memoryLastLayer- tensorSplit[g.i]++- layerCount++- break- }- }- }-- if layerCount < int(f.KV().BlockCount())+1 {- fullyLoaded = false- overflow += memoryLastLayer- }- }-- // Add the applicable (full or partial) graph allocations- for i := range gpus {- if tensorSplit[i] <= 0 {- continue- }- if fullyLoaded {- gpuAllocations[i] += graphFullOffload- } else {- gpuAllocations[i] += graphPartialOffload- }- }- if fullyLoaded {- graphOffload = graphFullOffload- } else {- graphOffload = graphPartialOffload- }-- // Summaries for the log- var memoryRequiredPartial, memoryRequiredTotal uint64- for i := range gpuAllocations {- memoryRequiredPartial += gpuAllocations[i]- }- memoryRequiredTotal = memoryRequiredPartial + overflow-- allocationsList := []string{}- for _, a := range gpuAllocations {- allocationsList = append(allocationsList, format.HumanBytes2(a))- }-- estimate := MemoryEstimate{- TotalSize: memoryRequiredTotal,- Layers: 0,- Graph: 0,- VRAMSize: 0,- GPUSizes: []uint64{},-- inferenceLibrary: strings.Join(libraries, ","),- layersRequested: opts.NumGPU,- layersModel: int(f.KV().BlockCount()) + 1,- availableList: availableList,- kv: kvTotal,- allocationsList: allocationsList,- memoryWeights: memoryWeights,- memoryLayerOutput: memoryLayerOutput,- graphFullOffload: graphFullOffload,- graphPartialOffload: graphPartialOffload,- projectorWeights: llamaEngineProjectorWeights + ollamaEngineProjectorWeights,- projectorGraph: ollamaEngineProjectorGraph,- }-- if len(gpus) == 0 {- return estimate- }- if layerCount == 0 {- slog.Debug("insufficient VRAM to load any model layers")- return estimate- }- estimate.Layers = layerCount- estimate.Graph = graphOffload- estimate.VRAMSize = memoryRequiredPartial- estimate.TotalSize = memoryRequiredTotal- estimate.TensorSplit = tensorSplit- estimate.GPUSizes = gpuAllocations- return estimate-}--func (m MemoryEstimate) LogValue() slog.Value {- attrs := []slog.Attr{- slog.String("library", m.inferenceLibrary),- slog.Group(- "layers",- // requested number of layers to offload- "requested", m.layersRequested,- // The number of layers the model has (including output)- "model", m.layersModel,- // estimated number of layers that can be offloaded- "offload", m.Layers,- // multi-gpu split for tensors- "split", m.TensorSplit,- ),- slog.Group(- "memory",- // memory available by GPU for offloading- "available", m.availableList,- "gpu_overhead", format.HumanBytes2(envconfig.GpuOverhead()),- slog.Group(- "required",- // memory required for full offloading- "full", format.HumanBytes2(m.TotalSize),- // memory required to offload layers.estimate layers- "partial", format.HumanBytes2(m.VRAMSize),- // memory of KV cache- "kv", format.HumanBytes2(m.kv),- // Allocations across the GPUs- "allocations", m.allocationsList,- ),- slog.Group(- "weights",- // memory of the weights- "total", format.HumanBytes2(m.memoryWeights+m.memoryLayerOutput),- // memory of repeating layers- "repeating", format.HumanBytes2(m.memoryWeights),- // memory of non-repeating layers- "nonrepeating", format.HumanBytes2(m.memoryLayerOutput),- ),- slog.Group(- "graph",- // memory of graph when fully offloaded- "full", format.HumanBytes2(m.graphFullOffload),- // memory of graph when not fully offloaded- "partial", format.HumanBytes2(m.graphPartialOffload),- ),- ),- }-- if m.projectorWeights > 0 {- attrs = append(attrs, slog.Group(- "projector",- "weights", format.HumanBytes2(m.projectorWeights),- "graph", format.HumanBytes2(m.projectorGraph),- ))- }-- return slog.GroupValue(attrs...)-}--func projectorMemoryRequirements(filename string) (weights uint64) {- file, err := os.Open(filename)- if err != nil {- return 0- }- defer file.Close()-- ggml, err := ggml.Decode(file, 1024)- if err != nil {- return 0- }-- for _, layer := range ggml.Tensors().GroupLayers() {- weights += layer.Size()- }-- return weights-}diff --git a/llm/memory_test.go b/llm/memory_test.godeleted file mode 100644index fce17b9c2..000000000--- a/llm/memory_test.go+++ /dev/null@@ -1,130 +0,0 @@-package llm--import (- "bytes"- "fmt"- "os"- "testing"-- "github.com/stretchr/testify/assert"- "github.com/stretchr/testify/require"-- "github.com/ollama/ollama/api"- "github.com/ollama/ollama/format"- "github.com/ollama/ollama/fs/ggml"- "github.com/ollama/ollama/ml"-)--func TestEstimateGPULayers(t *testing.T) {- t.Setenv("OLLAMA_DEBUG", "1")- t.Setenv("OLLAMA_KV_CACHE_TYPE", "") // Ensure default f16- t.Setenv("OLLAMA_CONTEXT_LENGTH", "2048")-- modelName := "dummy"- f, err := os.CreateTemp(t.TempDir(), modelName)- require.NoError(t, err)- defer f.Close()- inputLayerCount := 5-- tensors := []*ggml.Tensor{- {Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- {Name: "blk.1.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- {Name: "blk.2.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- {Name: "blk.3.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- {Name: "blk.4.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- {Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- }- assert.Len(t, tensors, inputLayerCount+1)- err = ggml.WriteGGUF(f, ggml.KV{- "general.architecture": "llama",- "llama.context_length": uint32(32),- "llama.embedding_length": uint32(4096),- "llama.block_count": uint32(inputLayerCount),- "llama.attention.head_count": uint32(32),- "llama.attention.head_count_kv": uint32(32),- "tokenizer.ggml.tokens": []string{" "},- "tokenizer.ggml.scores": []float32{0},- "tokenizer.ggml.token_type": []int32{0},- }, tensors)- require.NoError(t, err)-- ggml, err := LoadModel(f.Name(), 0)- if err != nil {- t.Fatal(err)- }-- // Simple CPU scenario- gpus := []ml.DeviceInfo{}- projectors := []string{}- opts := api.DefaultOptions()- t.Run("cpu", func(t *testing.T) {- estimate := estimateGPULayers(gpus, ggml, projectors, opts, 1)- assert.Equal(t, 0, estimate.Layers)- assert.Equal(t, uint64(0), estimate.Graph)- })-- // derived from the dummy ggml file above- graphPartialOffload := uint64(202377216)- graphFullOffload := uint64(171968512)- layerSize := uint64(33554436)- projectorSize := uint64(0)- memoryLayerOutput := uint64(4)-- // Dual CUDA scenario with asymmetry- gpuMinimumMemory := uint64(457 * format.MebiByte)- gpus = []ml.DeviceInfo{- {- DeviceID: ml.DeviceID{- Library: "CUDA",- },- },- {- DeviceID: ml.DeviceID{- Library: "CUDA",- },- },- }- // Nested array: GPU0 layer space, GPU1 layer space, expected gpu0, expected gpu1- for i, s := range []struct {- layer0, layer1 uint64- expect0, expect1 int- }{- {1, 1, 1, 1},- {2, 1, 2, 1},- {2, 2, 2, 2},- {1, 2, 1, 2},- {3, 3, 3, 3},- {4, 4, 3, 3},- {6, 6, 3, 3},- {0, 3, 0, 3},- } {- t.Run(fmt.Sprintf("%v", s), func(t *testing.T) {- gpus[0].FreeMemory = 0- gpus[1].FreeMemory = 0- gpus[0].FreeMemory += projectorSize- if s.layer0 > 0 {- gpus[0].FreeMemory += memoryLayerOutput- } else {- gpus[1].FreeMemory += memoryLayerOutput- }- gpus[0].FreeMemory += gpuMinimumMemory + layerSize + s.layer0*layerSize + 1- gpus[1].FreeMemory += gpuMinimumMemory + layerSize + s.layer1*layerSize + 1- gpus[0].FreeMemory += max(graphFullOffload, graphPartialOffload)- gpus[1].FreeMemory += max(graphFullOffload, graphPartialOffload)- estimate := estimateGPULayers(gpus, ggml, projectors, opts, 1)- assert.Equal(t, s.expect0+s.expect1, estimate.Layers, "scenario %d: %v", i, s)- assert.Equal(t, []int{s.expect0, s.expect1}, estimate.TensorSplit, "scenario %d: %v", i, s)- var layerSums uint64- for _, b := range estimate.GPUSizes {- layerSums += b- }- if estimate.Layers < inputLayerCount+1 {- assert.Less(t, estimate.VRAMSize, estimate.TotalSize, "scenario %d: %v %+v", i, s, estimate)- assert.Equal(t, estimate.VRAMSize, layerSums, "scenario %d: %v %+v", i, s, estimate)- } else {- assert.Equal(t, estimate.VRAMSize, estimate.TotalSize, "scenario %d: %v %+v", i, s, estimate)- assert.Equal(t, estimate.TotalSize, layerSums, "scenario %d: %v %+v", i, s, estimate)- }- })- }-}diff --git a/llm/server.go b/llm/server.goindex 87f97a010..83690fcdc 100644--- a/llm/server.go+++ b/llm/server.go@@ -92,7 +92,8 @@ type llmServer struct {numParallel intmodelPath string- loadRequest LoadRequest // Parameters used to initialize the runner+ loadRequest LoadRequest // Parameters used to initialize the runner+ mem *ml.BackendMemory // Memory allocations for this model// llamaModel is an instance of the cgo llama.cpp model definition// nil if this server is running the new engine@@ -113,15 +114,11 @@ type llmServer struct {type llamaServer struct {llmServer- ggml *ggml.GGML- gpus []ml.DeviceInfo // The set of GPUs covered by the memory estimate- estimate MemoryEstimate+ ggml *ggml.GGML}type ollamaServer struct {llmServer-- mem *ml.BackendMemory}// LoadModel will load a model from disk. The model must be in the GGML format.@@ -463,169 +460,226 @@ type LoadResponse struct {var ErrLoadRequiredFull = errors.New("unable to load full model on GPU")-func (s *llamaServer) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {- systemTotalMemory := systemInfo.TotalMemory- systemFreeMemory := systemInfo.FreeMemory- systemSwapFreeMemory := systemInfo.FreeSwap- slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))+func (s *llamaServer) Load(ctx context.Context, systemInfo ml.SystemInfo, systemGPUs []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {+ slog.Info("loading model", "model layers", s.totalLayers, "requested", s.options.NumGPU)++ gpus := append(make([]ml.DeviceInfo, 0, len(systemGPUs)), systemGPUs...)++ // Synthesize memory allocation information based on our estimates+ s.mem = &ml.BackendMemory{CPU: ml.DeviceMemory{+ Name: "CPU",+ Weights: make([]uint64, s.totalLayers),+ Cache: make([]uint64, s.totalLayers),+ }, GPUs: make([]ml.DeviceMemory, len(gpus))}++ for i := range s.mem.GPUs {+ s.mem.GPUs[i].Name = gpus[i].Name+ s.mem.GPUs[i].DeviceID = gpus[i].DeviceID+ s.mem.GPUs[i].Weights = make([]uint64, s.totalLayers)+ s.mem.GPUs[i].Cache = make([]uint64, s.totalLayers)+ }- if len(gpus) == 0 || s.options.NumGPU == 0 {- if !verifyCPUFit(s.ggml, s.modelPath, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, systemInfo, s.numParallel) {- slog.Info("model requires more memory than is currently available, evicting a model to make space", "estimate", s.estimate)- return nil, fmt.Errorf("model requires more system memory than is currently available %w", ErrLoadRequiredFull)+ kv, graphPartialOffload, graphFullOffload := s.ggml.GraphSize(uint64(s.options.NumCtx), uint64(s.loadRequest.BatchSize),+ s.loadRequest.Parallel, s.loadRequest.KvCacheType, s.loadRequest.FlashAttention)++ // Use the size of one layer as a buffer+ layers := s.ggml.Tensors().GroupLayers()+ if blk0, ok := layers["blk.0"]; ok {+ for i := range gpus {+ gpus[i].FreeMemory -= blk0.Size() + kv[0]}} else {- g := pickBestFullFitByLibrary(s.ggml, s.modelPath, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)- if g == nil {- if !requireFull {- g = pickBestPartialFitByLibrary(s.ggml, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)- } else {- slog.Info("model requires more memory than is currently available, evicting a model to make space", "estimate", s.estimate)- return nil, ErrLoadRequiredFull- }+ slog.Warn("model missing blk.0 layer size")+ }++ // Assign all the layers to the CPU for now, they will get reassigned later+ for i := range s.ggml.KV().BlockCount() {+ if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {+ s.mem.CPU.Weights[i] = blk.Size()+ s.mem.CPU.Cache[i] += kv[i]}- gpus = g}- s.estimate = estimateGPULayers(gpus, s.ggml, []string{s.loadRequest.ProjectorPath}, s.options, s.numParallel)+ // We historically haven't included InputWeights in the model size+ var outputWeights uint64+ if layer, ok := layers["output_norm"]; ok {+ outputWeights += layer.Size()+ }+ if layer, ok := layers["output"]; ok {+ outputWeights += layer.Size()+ } else if layer, ok := layers["token_embd"]; ok {+ outputWeights += layer.Size()+ }+ s.mem.CPU.Weights[s.totalLayers-1] = outputWeights- if len(gpus) >= 1 {- switch {- case s.options.NumGPU == 0:- gpus = []ml.DeviceInfo{}- case gpus[0].Library == "Metal" && s.estimate.VRAMSize > systemInfo.TotalMemory:- // disable partial offloading when model is greater than total system memory as this- // can lead to locking up the system- s.options.NumGPU = 0- gpus = []ml.DeviceInfo{}- case gpus[0].Library != "Metal" && s.estimate.Layers == 0:- // Don't bother loading into the GPU if no layers can fit- gpus = []ml.DeviceInfo{}- case s.options.NumGPU < 0 && s.estimate.Layers > 0:- s.options.NumGPU = s.estimate.Layers+ // The vision projector is always loaded on the first GPU if available.+ // This can't be assigned by us, so just subtract it from free space+ projectorGPU := -1+ var projectorWeights uint64+ if len(gpus) > 0 {+ for _, projector := range s.loadRequest.LoraPath {+ projectorWeights += projectorMemoryRequirements(projector)}- } else {- s.options.NumGPU = 0++ // llama.cpp uses the first discrete GPU if available, otherwise the first iGPU+ firstIntegrated := -1+ for i := range gpus {+ if !gpus[i].Integrated {+ projectorGPU = i+ break+ }+ if firstIntegrated == -1 {+ firstIntegrated = i+ }+ }+ if projectorGPU == -1 {+ projectorGPU = firstIntegrated+ }++ gpus[projectorGPU].FreeMemory -= projectorWeights}- // On linux and windows, over-allocating CPU memory will almost always result in an error- // Darwin has fully dynamic swap so has no direct concept of free swap space- if runtime.GOOS != "darwin" {- systemMemoryRequired := s.estimate.TotalSize - s.estimate.VRAMSize- available := systemInfo.FreeMemory + systemInfo.FreeSwap- if systemMemoryRequired > available {- slog.Warn("model request too large for system", "requested", format.HumanBytes2(systemMemoryRequired), "available", format.HumanBytes2(available), "total", format.HumanBytes2(systemInfo.TotalMemory), "free", format.HumanBytes2(systemInfo.FreeMemory), "swap", format.HumanBytes2(systemInfo.FreeSwap))- return nil, fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(systemMemoryRequired), format.HumanBytes2(available))+ var kvTotal uint64+ for _, kvLayer := range kv {+ kvTotal += kvLayer+ }++ if graphPartialOffload == 0 {+ headsKV := s.ggml.KV().HeadCountKVMin()+ if headsKV == 0 {+ headsKV = 1}+ gqa := s.ggml.KV().HeadCountMax() / headsKV+ graphPartialOffload = gqa * kvTotal / 6+ }+ if graphFullOffload == 0 {+ graphFullOffload = graphPartialOffload}- slog.Info("offload", "", s.estimate)+ // On Metal there's no partial offload overhead+ if len(gpus) > 0 && gpus[0].Library == "Metal" {+ graphPartialOffload = graphFullOffload+ }- s.gpus = gpus- s.loadRequest.GPULayers = createGPULayers(s.estimate, s.ggml, gpus, s.options.NumGPU)+ // Create a layout based on the memory data that we've built. The compute graph+ // for GPUs is iteratively assigned based on the number of GPUs that are required.+ var gpuLayers ml.GPULayersList+ for {+ prevGPULayers := gpuLayers- // Mmap is only supported on the llama engine- if s.textProcessor == nil {- s.loadRequest.UseMmap = true+ var err error+ gpuLayers, err = s.createLayout(systemInfo, gpus, s.mem, requireFull, 0)+ if err != nil {+ return nil, err+ }- // mmap has issues with partial offloading on metal- for _, g := range gpus {- if g.Library == "Metal" &&- uint64(s.options.NumGPU) > 0 &&- uint64(s.options.NumGPU) < s.ggml.KV().BlockCount()+1 {- s.options.UseMMap = new(bool)- *s.options.UseMMap = false+ if len(gpuLayers) > len(prevGPULayers) {+ for _, gl := range gpuLayers {+ for i := range s.mem.GPUs {+ if gl.DeviceID == s.mem.GPUs[i].DeviceID {+ s.mem.GPUs[i].Graph = max(graphPartialOffload, graphFullOffload)+ break+ }+ }}+ } else {+ break}+ }++ // This maintains the historical assignment of graph sizes, though it isn't fully accurate+ graphSize := graphFullOffload+ if gpuLayers.Sum() < int(s.totalLayers) {+ graphSize = graphPartialOffload+ }- // Windows CUDA should not use mmap for best performance- // Linux with a model larger than free space, mmap leads to thrashing- // For CPU loads we want the memory to be allocated, not FS cache- if (runtime.GOOS == "windows" && len(gpus) > 0 && gpus[0].Library == "CUDA" && s.options.UseMMap == nil) ||- (runtime.GOOS == "linux" && systemInfo.FreeMemory < s.estimate.TotalSize && s.options.UseMMap == nil) ||- (len(gpus) == 0 && s.options.UseMMap == nil) ||- (len(gpus) > 0 && gpus[0].Library == "Vulkan" && s.options.UseMMap == nil) ||- (s.options.UseMMap != nil && !*s.options.UseMMap) {- s.loadRequest.UseMmap = false+ // For all layers that we have assigned to GPUs, move them in the memory data so+ // that it is reported accurately+ for _, gl := range gpuLayers {+ for i := range s.mem.GPUs {+ if gl.DeviceID == s.mem.GPUs[i].DeviceID {+ for _, l := range gl.Layers {+ s.mem.GPUs[i].Weights[l] = s.mem.CPU.Weights[l]+ s.mem.GPUs[i].Cache[l] = s.mem.CPU.Cache[l]++ s.mem.CPU.Weights[l] = 0+ s.mem.CPU.Cache[l] = 0+ }++ s.mem.GPUs[i].Graph = graphSize+ break+ }}}+ if projectorGPU > 0 && len(s.mem.GPUs[projectorGPU].Weights) > 0 {+ s.mem.GPUs[projectorGPU].Weights[s.totalLayers-1] += projectorWeights+ }++ slog.Debug("memory", "estimate", s.mem)+ s.mem.Log(slog.LevelInfo)++ // The llama engine uses mmap by default+ s.loadRequest.UseMmap = true++ // mmap has issues with partial offloading on metal+ for _, g := range gpus {+ if g.Library == "Metal" &&+ uint64(s.options.NumGPU) > 0 &&+ uint64(s.options.NumGPU) < s.totalLayers {+ s.options.UseMMap = new(bool)+ *s.options.UseMMap = false+ }+ }++ // Windows CUDA should not use mmap for best performance+ // Linux with a model larger than free space, mmap leads to thrashing+ // For CPU loads we want the memory to be allocated, not FS cache+ if (runtime.GOOS == "windows" && len(gpus) > 0 && gpus[0].Library == "CUDA" && s.options.UseMMap == nil) ||+ (runtime.GOOS == "linux" && systemInfo.FreeMemory < s.TotalSize() && s.options.UseMMap == nil) ||+ (len(gpus) == 0 && s.options.UseMMap == nil) ||+ (len(gpus) > 0 && gpus[0].Library == "Vulkan" && s.options.UseMMap == nil) ||+ (s.options.UseMMap != nil && !*s.options.UseMMap) {+ s.loadRequest.UseMmap = false+ }+if err := s.waitUntilRunnerLaunched(ctx); err != nil {return nil, err}+ s.loadRequest.GPULayers = gpuLayersresp, err := s.initModel(ctx, s.loadRequest, LoadOperationCommit)if err != nil {return nil, err}- // On the Ollama engine, we can print out a summary of the memory allocations.- // We don't have this for the llama engine but it does something similar itself.- if s.textProcessor != nil {- resp.Memory.Log(slog.LevelInfo)- }-if !resp.Success {- slog.Warn("failed to allocate memory for model", "memory", resp.Memory)return nil, errors.New("failed to allocate memory for model")}// The llama engine does its memory allocations together with model loading, so we// need to wait until it is done to ensure that we have accurate memory data before// loading the next model- if s.textProcessor == nil {- return uniqueDeviceIDs(s.loadRequest.GPULayers), s.WaitUntilRunning(ctx)- } else {- return uniqueDeviceIDs(s.loadRequest.GPULayers), nil- }+ return uniqueDeviceIDs(s.loadRequest.GPULayers), s.WaitUntilRunning(ctx)}-// createGPULayers maps from the tensor splits assigned by the memory estimates to explicit assignment-// of particular layers onto GPUs-func createGPULayers(estimate MemoryEstimate, ggml *ggml.GGML, gpus []ml.DeviceInfo, numGPU int) ml.GPULayersList {- if numGPU <= 0 || len(gpus) == 0 {- return nil- }-- gpuLayers := make(ml.GPULayersList, len(gpus))- for i := range gpuLayers {- gpuLayers[i].DeviceID = gpus[i].DeviceID- }-- var sum float32- splits := make([]float32, len(estimate.TensorSplit))- // cumulative sum of all splits- for i := range splits {- sum += float32(estimate.TensorSplit[i])- splits[i] = sum+func projectorMemoryRequirements(filename string) (weights uint64) {+ file, err := os.Open(filename)+ if err != nil {+ return 0}+ defer file.Close()- if sum <= 0 {- return nil+ ggml, err := ggml.Decode(file, 1024)+ if err != nil {+ return 0}- // normalize splits- for i := range splits {- splits[i] /= sum+ for _, layer := range ggml.Tensors().GroupLayers() {+ weights += layer.Size()}- blocks := int(ggml.KV().BlockCount())- gpuRangeStart := max(0, blocks-numGPU)- gpuRangeStop := min(gpuRangeStart+numGPU, blocks+1)- for i := range blocks + 1 {- if i < gpuRangeStart || i >= gpuRangeStop {- continue- }-- index := slices.IndexFunc(splits, func(f float32) bool { return float32(i-gpuRangeStart)/float32(gpuRangeStop-gpuRangeStart) < f })- if index < 0 || index >= len(gpus) {- continue- }-- gpuLayers[index].Layers = append(gpuLayers[index].Layers, i)- }-- return gpuLayers+ return weights}// Load finds the optimal layout of layers to offload on GPUs based on no initial information about the size of the model@@ -652,23 +706,6 @@ func (s *ollamaServer) Load(ctx context.Context, systemInfo ml.SystemInfo, gpusslog.Info("loading model", "model layers", s.totalLayers, "requested", s.options.NumGPU)- systemTotalMemory := systemInfo.TotalMemory- systemFreeMemory := systemInfo.FreeMemory- systemSwapFreeMemory := systemInfo.FreeSwap- slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))-- for _, gpu := range gpus {- available := gpu.FreeMemory - envconfig.GpuOverhead() - gpu.MinimumMemory()- if gpu.FreeMemory < envconfig.GpuOverhead()+gpu.MinimumMemory() {- available = 0- }- slog.Info("gpu memory", "id", gpu.ID, "library", gpu.Library,- "available", format.HumanBytes2(available),- "free", format.HumanBytes2(gpu.FreeMemory),- "minimum", format.HumanBytes2(gpu.MinimumMemory()),- "overhead", format.HumanBytes2(envconfig.GpuOverhead()))- }-pastAllocations := make(map[uint64]struct{})var backoff float32@@ -834,25 +871,22 @@ func uniqueDeviceIDs(gpuLayers ml.GPULayersList) []ml.DeviceID {// - Calculating how much space each GPU has available for layers, based on free memory and space occupied by the graph// - Assigning layers// - Ensuring that we don't exceed limits, such as requirements about partial offloading or system memory-func (s *ollamaServer) createLayout(systemInfo ml.SystemInfo, systemGPUs []ml.DeviceInfo, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, error) {+func (s *llmServer) createLayout(systemInfo ml.SystemInfo, systemGPUs []ml.DeviceInfo, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, error) {if memory == nil {memory = &ml.BackendMemory{CPU: ml.DeviceMemory{Weights: make([]uint64, s.totalLayers),Cache: make([]uint64, s.totalLayers),}}}- gpuLayers, layers, err := s.buildLayout(systemGPUs, memory, requireFull, backoff)- if err != nil {- return nil, err- }- err = s.verifyLayout(systemInfo, memory, requireFull, gpuLayers, layers)+ gpuLayers, layers := s.buildLayout(systemGPUs, memory, requireFull, backoff)+ err := s.verifyLayout(systemInfo, memory, requireFull, gpuLayers, layers)if err != nil {return nil, err}return gpuLayers, nil}-func (s *ollamaServer) buildLayout(systemGPUs []ml.DeviceInfo, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, []uint64, error) {+func (s *llmServer) buildLayout(systemGPUs []ml.DeviceInfo, memory *ml.BackendMemory, requireFull bool, backoff float32) (ml.GPULayersList, []uint64) {gpus := append(make([]ml.DeviceInfo, 0, len(systemGPUs)), systemGPUs...)sort.Sort(sort.Reverse(ml.ByFreeMemory(gpus)))@@ -910,11 +944,11 @@ func (s *ollamaServer) buildLayout(systemGPUs []ml.DeviceInfo, memory *ml.BackengpuLayers = libraryGpuLayers}}- return gpuLayers, layers, nil+ return gpuLayers, layers}// verifyLayout ensures that we don't exceed limits, such as requirements about partial offloading or system memory-func (s *ollamaServer) verifyLayout(systemInfo ml.SystemInfo, memory *ml.BackendMemory, requireFull bool, gpuLayers ml.GPULayersList, layers []uint64) error {+func (s *llmServer) verifyLayout(systemInfo ml.SystemInfo, memory *ml.BackendMemory, requireFull bool, gpuLayers ml.GPULayersList, layers []uint64) error {// These sizes will only increase as we go through additional iterations and get additional information.cpuSize := memory.InputWeights + memory.CPU.Graphvar vramSize uint64@@ -942,11 +976,13 @@ nextLayer:if requireFull {if gpuLayers.Sum() < len(layers) && (s.options.NumGPU < 0 || gpuLayers.Sum() < s.options.NumGPU) {+ slog.Info("model requires more memory than is currently available, evicting a model to make space", "loaded layers", gpuLayers.Sum())return ErrLoadRequiredFull}if cpuSize > systemInfo.FreeMemory {- return ErrLoadRequiredFull+ slog.Info("model requires more system memory than is currently available, evicting a model to make space", "required", cpuSize, "free", systemInfo.FreeMemory)+ return fmt.Errorf("model requires more system memory than is currently available %w", ErrLoadRequiredFull)}}@@ -1734,31 +1770,12 @@ func (s *llmServer) Close() error {return nil}-func (s *llamaServer) VRAMSize() uint64 {- return s.estimate.VRAMSize-}--func (s *llamaServer) TotalSize() uint64 {- return s.estimate.TotalSize-}--func (s *llamaServer) VRAMByGPU(id ml.DeviceID) uint64 {- for i, gpu := range s.gpus {- if gpu.DeviceID == id {- if i < len(s.estimate.GPUSizes) {- return s.estimate.GPUSizes[i]- }- }- }- return 0-}-func (s *llamaServer) GetDeviceInfos(ctx context.Context) []ml.DeviceInfo {slog.Debug("llamarunner free vram reporting not supported")return nil}-func (s *ollamaServer) VRAMSize() uint64 {+func (s *llmServer) VRAMSize() uint64 {if s.mem == nil {return 0}@@ -1786,7 +1803,7 @@ func (s *ollamaServer) VRAMSize() uint64 {return mem}-func (s *ollamaServer) TotalSize() uint64 {+func (s *llmServer) TotalSize() uint64 {if s.mem == nil {return 0}@@ -1800,7 +1817,7 @@ func (s *ollamaServer) TotalSize() uint64 {return mem}-func (s *ollamaServer) VRAMByGPU(id ml.DeviceID) uint64 {+func (s *llmServer) VRAMByGPU(id ml.DeviceID) uint64 {if s.mem == nil {return 0}diff --git a/server/sched.go b/server/sched.goindex 5ae42efd1..c5bc6692d 100644--- a/server/sched.go+++ b/server/sched.go@@ -437,6 +437,23 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, systemInfo ml.SystemInfos.loadedMu.Unlock()+ systemTotalMemory := systemInfo.TotalMemory+ systemFreeMemory := systemInfo.FreeMemory+ systemSwapFreeMemory := systemInfo.FreeSwap+ slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))++ for _, gpu := range gpus {+ available := gpu.FreeMemory - envconfig.GpuOverhead() - gpu.MinimumMemory()+ if gpu.FreeMemory < envconfig.GpuOverhead()+gpu.MinimumMemory() {+ available = 0+ }+ slog.Info("gpu memory", "id", gpu.ID, "library", gpu.Library,+ "available", format.HumanBytes2(available),+ "free", format.HumanBytes2(gpu.FreeMemory),+ "minimum", format.HumanBytes2(gpu.MinimumMemory()),+ "overhead", format.HumanBytes2(envconfig.GpuOverhead()))+ }+gpuIDs, err := llama.Load(req.ctx, systemInfo, gpus, requireFull)if err != nil {if errors.Is(err, llm.ErrLoadRequiredFull) {