Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
kvcache: Use Cast instead of Copy for flash attention masks
Jesse Gross05ccb17c6e10
diff --git a/kvcache/causal.go b/kvcache/causal.goindex 96d8067eb..31f552331 100644--- a/kvcache/causal.go+++ b/kvcache/causal.go@@ -378,9 +378,7 @@ func (c *Causal) buildMask(ctx ml.Context) ml.Tensor {maskTensor := ctx.Input().FromFloatSlice(mask, length, batchSize)if c.config.MaskDType != ml.DTypeF32 {- out := ctx.Input().Empty(c.config.MaskDType, maskTensor.Shape()...)- ctx.Forward(maskTensor.Copy(ctx, out))- maskTensor = out+ maskTensor = maskTensor.Cast(ctx, c.config.MaskDType)}return maskTensordiff --git a/ml/backend.go b/ml/backend.goindex 638a05d14..705724821 100644--- a/ml/backend.go+++ b/ml/backend.go@@ -396,6 +396,7 @@ type Tensor interface {Shape() []intDType() DType+ Cast(ctx Context, dtype DType) TensorBytes() []byteFloats() []float32diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.goindex f8121cc53..13d898aad 100644--- a/ml/backend/ggml/ggml.go+++ b/ml/backend/ggml/ggml.go@@ -843,23 +843,7 @@ func (c *Context) newTensor(dtype ml.DType, shape []int) ml.Tensor {panic("set Input or Layer before creating tensors")}- var cdtype uint32- switch dtype {- case ml.DTypeF32:- cdtype = C.GGML_TYPE_F32- case ml.DTypeF16:- cdtype = C.GGML_TYPE_F16- case ml.DTypeQ80:- cdtype = C.GGML_TYPE_Q8_0- case ml.DTypeQ40:- cdtype = C.GGML_TYPE_Q4_0- case ml.DTypeI32:- cdtype = C.GGML_TYPE_I32- case ml.DTypeMXFP4:- cdtype = C.GGML_TYPE_MXFP4- default:- panic("unsupported dtype")- }+ cdtype := ggmlDType(dtype)if len(shape) < 1 || shape[0] == 0 {var shape C.int64_t = 0@@ -1056,6 +1040,32 @@ func (t *Tensor) DType() ml.DType {}}+func ggmlDType(dtype ml.DType) uint32 {+ switch dtype {+ case ml.DTypeF32:+ return C.GGML_TYPE_F32+ case ml.DTypeF16:+ return C.GGML_TYPE_F16+ case ml.DTypeQ80:+ return C.GGML_TYPE_Q8_0+ case ml.DTypeQ40:+ return C.GGML_TYPE_Q4_0+ case ml.DTypeI32:+ return C.GGML_TYPE_I32+ case ml.DTypeMXFP4:+ return C.GGML_TYPE_MXFP4+ default:+ panic("unsupported dtype")+ }+}++func (t *Tensor) Cast(ctx ml.Context, dtype ml.DType) ml.Tensor {+ return &Tensor{+ b: t.b,+ t: C.ggml_cast(ctx.(*Context).ctx, t.t, ggmlDType(dtype)),+ }+}+func (t *Tensor) Neg(ctx ml.Context) ml.Tensor {return &Tensor{b: t.b,