Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

ggml: fix crash for array head counts

Devon Rifkin6ed88985903b
diff --git a/fs/ggml/ggml.go b/fs/ggml/ggml.go
index 427a43aec..0d38f29e8 100644
--- a/fs/ggml/ggml.go
+++ b/fs/ggml/ggml.go
@@ -33,7 +33,8 @@ func (kv KV) Kind() string {
}
func (kv KV) ParameterCount() uint64 {
- return keyValue(kv, "general.parameter_count", uint64(0))
+ val, _ := keyValue(kv, "general.parameter_count", uint64(0))
+ return val
}
func (kv KV) FileType() fileType {
@@ -52,16 +53,27 @@ func (kv KV) EmbeddingLength() uint64 {
return uint64(kv.Uint("embedding_length"))
}
-func (kv KV) HeadCount() uint64 {
- return uint64(kv.Uint("attention.head_count"))
+func (kv KV) HeadCountMax() uint64 {
+ // TODO(drifkin): using the max value can cause an overestimation. In the
+ // future if array values become more popular, we can adapt the more invasive
+ // <https://github.com/ollama/ollama/pull/10225>
+ return uint64(kv.UintOrMaxArrayValue("attention.head_count", 1))
}
-func (kv KV) HeadCountKV() uint64 {
- return uint64(kv.Uint("attention.head_count_kv", 1))
+func (kv KV) HeadCountMin() uint64 {
+ return uint64(kv.UintOrMinArrayValue("attention.head_count", 1))
}
-func (kv KV) EmbeddingHeadCount() uint64 {
- if heads := kv.HeadCount(); heads > 0 {
+func (kv KV) HeadCountKVMax() uint64 {
+ return uint64(kv.UintOrMaxArrayValue("attention.head_count_kv", 1))
+}
+
+func (kv KV) HeadCountKVMin() uint64 {
+ return uint64(kv.UintOrMinArrayValue("attention.head_count_kv", 1))
+}
+
+func (kv KV) EmbeddingHeadCountMax() uint64 {
+ if heads := kv.HeadCountMin(); heads > 0 {
return kv.EmbeddingLength() / heads
}
@@ -69,15 +81,11 @@ func (kv KV) EmbeddingHeadCount() uint64 {
}
func (kv KV) EmbeddingHeadCountK() uint64 {
- return uint64(kv.Uint("attention.key_length", uint32(kv.EmbeddingHeadCount())))
+ return uint64(kv.Uint("attention.key_length", uint32(kv.EmbeddingHeadCountMax())))
}
func (kv KV) EmbeddingHeadCountV() uint64 {
- return uint64(kv.Uint("attention.value_length", uint32(kv.EmbeddingHeadCount())))
-}
-
-func (kv KV) GQA() uint64 {
- return kv.HeadCount() / kv.HeadCountKV()
+ return uint64(kv.Uint("attention.value_length", uint32(kv.EmbeddingHeadCountMax())))
}
func (kv KV) ContextLength() uint64 {
@@ -89,35 +97,72 @@ func (kv KV) ChatTemplate() string {
}
func (kv KV) String(key string, defaultValue ...string) string {
- return keyValue(kv, key, append(defaultValue, "")...)
+ val, _ := keyValue(kv, key, append(defaultValue, "")...)
+ return val
}
func (kv KV) Uint(key string, defaultValue ...uint32) uint32 {
- return keyValue(kv, key, append(defaultValue, 0)...)
+ val, _ := keyValue(kv, key, append(defaultValue, 0)...)
+ return val
}
func (kv KV) Float(key string, defaultValue ...float32) float32 {
- return keyValue(kv, key, append(defaultValue, 0)...)
+ val, _ := keyValue(kv, key, append(defaultValue, 0)...)
+ return val
}
func (kv KV) Bool(key string, defaultValue ...bool) bool {
- return keyValue(kv, key, append(defaultValue, false)...)
+ val, _ := keyValue(kv, key, append(defaultValue, false)...)
+ return val
+}
+
+func (kv KV) UintOrMaxArrayValue(key string, defaultValue uint32) uint32 {
+ _, max := kv.UintOrArrayValue(key, defaultValue)
+ return max
+}
+
+func (kv KV) UintOrMinArrayValue(key string, defaultValue uint32) uint32 {
+ min, _ := kv.UintOrArrayValue(key, defaultValue)
+ return min
+}
+
+func (kv KV) UintOrArrayValue(key string, defaultValue uint32) (uint32, uint32) {
+ if u32, ok := keyValue(kv, key, uint32(0)); ok {
+ return u32, u32
+ } else if u32s, ok := keyValue(kv, key, &array[uint32]{}); ok {
+ min := slices.Min(u32s.values)
+ max := slices.Max(u32s.values)
+ return min, max
+ } else if i32s, ok := keyValue(kv, key, &array[int32]{}); ok {
+ min := slices.Min(i32s.values)
+ max := slices.Max(i32s.values)
+ if min < 0 || max < 0 {
+ slog.Warn("array values are unexpectedly negative", "key", key, "min", min, "max", max)
+ }
+ return uint32(min), uint32(max)
+ }
+
+ return defaultValue, defaultValue
}
func (kv KV) Strings(key string, defaultValue ...[]string) []string {
- return keyValue(kv, key, &array[string]{values: append(defaultValue, []string(nil))[0]}).values
+ val, _ := keyValue(kv, key, &array[string]{values: append(defaultValue, []string(nil))[0]})
+ return val.values
}
func (kv KV) Ints(key string, defaultValue ...[]int32) []int32 {
- return keyValue(kv, key, &array[int32]{values: append(defaultValue, []int32(nil))[0]}).values
+ val, _ := keyValue(kv, key, &array[int32]{values: append(defaultValue, []int32(nil))[0]})
+ return val.values
}
func (kv KV) Uints(key string, defaultValue ...[]uint32) []uint32 {
- return keyValue(kv, key, &array[uint32]{values: append(defaultValue, []uint32(nil))[0]}).values
+ val, _ := keyValue(kv, key, &array[uint32]{values: append(defaultValue, []uint32(nil))[0]})
+ return val.values
}
func (kv KV) Floats(key string, defaultValue ...[]float32) []float32 {
- return keyValue(kv, key, &array[float32]{values: append(defaultValue, []float32(nil))[0]}).values
+ val, _ := keyValue(kv, key, &array[float32]{values: append(defaultValue, []float32(nil))[0]})
+ return val.values
}
func (kv KV) OllamaEngineRequired() bool {
@@ -140,17 +185,17 @@ type arrayValueTypes interface {
*array[string] | *array[float32] | *array[float64] | *array[bool]
}
-func keyValue[T valueTypes | arrayValueTypes](kv KV, key string, defaultValue ...T) T {
+func keyValue[T valueTypes | arrayValueTypes](kv KV, key string, defaultValue ...T) (T, bool) {
if !strings.HasPrefix(key, "tokenizer.") && !strings.HasPrefix(key, "general.") {
key = kv.Architecture() + "." + key
}
- if val, ok := kv[key]; ok {
- return val.(T)
+ if val, ok := kv[key].(T); ok {
+ return val, true
}
- slog.Warn("key not found", "key", key, "default", defaultValue[0])
- return defaultValue[0]
+ slog.Warn("key with type not found", "key", key, "default", defaultValue[0])
+ return defaultValue[0], false
}
type Tensors struct {
@@ -413,11 +458,11 @@ func Decode(rs io.ReadSeeker, maxArraySize int) (*GGML, int64, error) {
func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType string) (kv []uint64, partialOffload, fullOffload uint64) {
embedding := f.KV().EmbeddingLength()
- heads := f.KV().HeadCount()
- headsKV := f.KV().HeadCountKV()
+ heads := f.KV().HeadCountMax()
+ headsKV := f.KV().HeadCountKVMax()
vocab := uint64(f.KV()["tokenizer.ggml.tokens"].(*array[string]).size)
- embeddingHeads := f.KV().EmbeddingHeadCount()
+ embeddingHeads := f.KV().EmbeddingHeadCountMax()
embeddingHeadsK := f.KV().EmbeddingHeadCountK()
embeddingHeadsV := f.KV().EmbeddingHeadCountV()
diff --git a/fs/ggml/ggml_test.go b/fs/ggml/ggml_test.go
index c1c1b43b0..225e74841 100644
--- a/fs/ggml/ggml_test.go
+++ b/fs/ggml/ggml_test.go
@@ -269,3 +269,33 @@ func TestKeyValue(t *testing.T) {
t.Errorf("unexpected uint8s (-got +want):\n%s", diff)
}
}
+
+func TestHeadCount(t *testing.T) {
+ valuesArray := []int32{1, 5, 3, 4}
+ cases := []struct {
+ kv KV
+ want uint64
+ }{
+ {
+ kv: KV{
+ "general.architecture": "abc",
+ "abc.attention.head_count": &array[int32]{values: valuesArray, size: len(valuesArray)},
+ },
+ want: uint64(5),
+ },
+ {
+ kv: KV{
+ "general.architecture": "abc",
+ "abc.attention.head_count": uint32(3),
+ },
+ want: uint64(3),
+ },
+ }
+
+ for _, tt := range cases {
+ got := tt.kv.HeadCountMax()
+ if got != tt.want {
+ t.Errorf("unexpected max value: got=%d want=%d", got, tt.want)
+ }
+ }
+}
diff --git a/llm/memory.go b/llm/memory.go
index e05327f79..d029e4d31 100644
--- a/llm/memory.go
+++ b/llm/memory.go
@@ -149,7 +149,12 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
}
if graphPartialOffload == 0 {
- graphPartialOffload = f.KV().GQA() * kvTotal / 6
+ headsKV := f.KV().HeadCountKVMin()
+ if headsKV == 0 {
+ headsKV = 1
+ }
+ gqa := f.KV().HeadCountMax() / headsKV
+ graphPartialOffload = gqa * kvTotal / 6
}
if graphFullOffload == 0 {
graphFullOffload = graphPartialOffload