Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

fix: quantization with template

Michael Yange835ef183691
diff --git a/server/images.go b/server/images.go
index 53a957715..98794149e 100644
--- a/server/images.go
+++ b/server/images.go
@@ -414,17 +414,22 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
return err
}
- layers, err := parseFromFile(ctx, temp, "", fn)
+ layer, err := NewLayer(temp, baseLayer.MediaType)
if err != nil {
return err
}
- if len(layers) != 1 {
- return errors.New("quantization failed")
+ if _, err := temp.Seek(0, io.SeekStart); err != nil {
+ return err
+ }
+
+ ggml, _, err := llm.DecodeGGML(temp)
+ if err != nil {
+ return err
}
- baseLayer.Layer = layers[0].Layer
- baseLayer.GGML = layers[0].GGML
+ baseLayer.Layer = layer
+ baseLayer.GGML = ggml
}
}