Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
server: improve tensor quantization fallback logic (#10806)
Bruce MacDonaldfbe6ae285a23
diff --git a/server/quantization.go b/server/quantization.goindex adfc948ec..e57e8a4da 100644--- a/server/quantization.go+++ b/server/quantization.go@@ -120,14 +120,30 @@ func getTensorNewType(kv fsggml.KV, qs *quantizeState, newType fsggml.TensorTypeif newType.IsQuantized() {nx := shape[0]- ny := uint64(1)- if len(shape) > 1 {- ny = shape[1]- }qk_k := newType.BlockSize()++ // Check if first dimension is divisible by block sizeif nx%qk_k != 0 {- slog.Warn(fmt.Sprintf("tensor cols %d x %d are not divisible by %d, required for %s. Falling back to quantization %s", nx, ny, qk_k, newType.String(), fsggml.TensorTypeF16.String()))- newType = fsggml.TensorTypeF16+ // Store the original type for logging+ originalType := newType++ // Select appropriate fallback based on original type+ switch newType {+ case fsggml.TensorTypeQ4_K:+ newType = fsggml.TensorTypeQ5_0+ case fsggml.TensorTypeQ5_K:+ newType = fsggml.TensorTypeQ5_1+ case fsggml.TensorTypeQ6_K:+ newType = fsggml.TensorTypeQ8_0+ }++ // Final check - if still incompatible, fall back to F16+ if nx%newType.BlockSize() != 0 {+ newType = fsggml.TensorTypeF16+ }++ slog.Warn(fmt.Sprintf("tensor cols %d are not divisible by %d, required for %s - using fallback quantization %s",+ nx, qk_k, originalType.String(), newType.String()))}}return newType