Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
gpt-oss (#11672)
Michael Yangfa7776fd2458
diff --git a/api/types.go b/api/types.goindex 699dba428..e2c63b622 100644--- a/api/types.go+++ b/api/types.go@@ -85,10 +85,11 @@ type GenerateRequest struct {Options map[string]any `json:"options"`// Think controls whether thinking/reasoning models will think before- // responding. Needs to be a pointer so we can distinguish between false+ // responding. Can be a boolean (true/false) or a string ("high", "medium", "low")+ // for supported models. Needs to be a pointer so we can distinguish between false// (request that thinking _not_ be used) and unset (use the old behavior// before this option was introduced)- Think *bool `json:"think,omitempty"`+ Think *ThinkValue `json:"think,omitempty"`}// ChatRequest describes a request sent by [Client.Chat].@@ -116,8 +117,9 @@ type ChatRequest struct {Options map[string]any `json:"options"`// Think controls whether thinking/reasoning models will think before- // responding- Think *bool `json:"think,omitempty"`+ // responding. Can be a boolean (true/false) or a string ("high", "medium", "low")+ // for supported models.+ Think *ThinkValue `json:"think,omitempty"`}type Tools []Tool@@ -508,6 +510,8 @@ type GenerateResponse struct {Context []int `json:"context,omitempty"`Metrics++ ToolCalls []ToolCall `json:"tool_calls,omitempty"`}// ModelDetails provides details about a model.@@ -677,6 +681,113 @@ func DefaultOptions() Options {}}+// ThinkValue represents a value that can be a boolean or a string ("high", "medium", "low")+type ThinkValue struct {+ // Value can be a bool or string+ Value interface{}+}++// IsValid checks if the ThinkValue is valid+func (t *ThinkValue) IsValid() bool {+ if t == nil || t.Value == nil {+ return true // nil is valid (means not set)+ }++ switch v := t.Value.(type) {+ case bool:+ return true+ case string:+ return v == "high" || v == "medium" || v == "low"+ default:+ return false+ }+}++// IsBool returns true if the value is a boolean+func (t *ThinkValue) IsBool() bool {+ if t == nil || t.Value == nil {+ return false+ }+ _, ok := t.Value.(bool)+ return ok+}++// IsString returns true if the value is a string+func (t *ThinkValue) IsString() bool {+ if t == nil || t.Value == nil {+ return false+ }+ _, ok := t.Value.(string)+ return ok+}++// AsBool returns the value as a bool (true if enabled in any way)+func (t *ThinkValue) AsBool() bool {+ if t == nil || t.Value == nil {+ return false+ }++ switch v := t.Value.(type) {+ case bool:+ return v+ case string:+ // Any string value ("high", "medium", "low") means thinking is enabled+ return v == "high" || v == "medium" || v == "low"+ default:+ return false+ }+}++// AsString returns the value as a string+func (t *ThinkValue) AsString() string {+ if t == nil || t.Value == nil {+ return ""+ }++ switch v := t.Value.(type) {+ case string:+ return v+ case bool:+ if v {+ return "medium" // Default level when just true+ }+ return ""+ default:+ return ""+ }+}++// UnmarshalJSON implements json.Unmarshaler+func (t *ThinkValue) UnmarshalJSON(data []byte) error {+ // Try to unmarshal as bool first+ var b bool+ if err := json.Unmarshal(data, &b); err == nil {+ t.Value = b+ return nil+ }++ // Try to unmarshal as string+ var s string+ if err := json.Unmarshal(data, &s); err == nil {+ // Validate string values+ if s != "high" && s != "medium" && s != "low" {+ return fmt.Errorf("invalid think value: %q (must be \"high\", \"medium\", \"low\", true, or false)", s)+ }+ t.Value = s+ return nil+ }++ return fmt.Errorf("think must be a boolean or string (\"high\", \"medium\", \"low\")")+}++// MarshalJSON implements json.Marshaler+func (t *ThinkValue) MarshalJSON() ([]byte, error) {+ if t == nil || t.Value == nil {+ return []byte("null"), nil+ }+ return json.Marshal(t.Value)+}+type Duration struct {time.Duration}diff --git a/api/types_test.go b/api/types_test.goindex 9c2fb1f11..841853808 100644--- a/api/types_test.go+++ b/api/types_test.go@@ -374,24 +374,21 @@ func TestPropertyType_MarshalJSON(t *testing.T) {}func TestThinking_UnmarshalJSON(t *testing.T) {- trueVal := true- falseVal := false-tests := []struct {name stringinput string- expectedThinking *bool+ expectedThinking *ThinkValueexpectedError bool}{{name: "true",input: `{ "think": true }`,- expectedThinking: &trueVal,+ expectedThinking: &ThinkValue{Value: true},},{name: "false",input: `{ "think": false }`,- expectedThinking: &falseVal,+ expectedThinking: &ThinkValue{Value: false},},{name: "unset",@@ -399,8 +396,23 @@ func TestThinking_UnmarshalJSON(t *testing.T) {expectedThinking: nil,},{- name: "invalid",- input: `{ "think": "true" }`,+ name: "string_high",+ input: `{ "think": "high" }`,+ expectedThinking: &ThinkValue{Value: "high"},+ },+ {+ name: "string_medium",+ input: `{ "think": "medium" }`,+ expectedThinking: &ThinkValue{Value: "medium"},+ },+ {+ name: "string_low",+ input: `{ "think": "low" }`,+ expectedThinking: &ThinkValue{Value: "low"},+ },+ {+ name: "invalid_string",+ input: `{ "think": "invalid" }`,expectedThinking: nil,expectedError: true,},@@ -414,7 +426,12 @@ func TestThinking_UnmarshalJSON(t *testing.T) {require.Error(t, err)} else {require.NoError(t, err)- assert.Equal(t, test.expectedThinking, req.Think)+ if test.expectedThinking == nil {+ assert.Nil(t, req.Think)+ } else {+ require.NotNil(t, req.Think)+ assert.Equal(t, test.expectedThinking.Value, req.Think.Value)+ }}})}diff --git a/cmd/cmd.go b/cmd/cmd.goindex 1d1d116ba..de3fc86a7 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -322,11 +322,23 @@ func RunHandler(cmd *cobra.Command, args []string) error {thinkFlag := cmd.Flags().Lookup("think")if thinkFlag.Changed {- think, err := cmd.Flags().GetBool("think")+ thinkStr, err := cmd.Flags().GetString("think")if err != nil {return err}- opts.Think = &think++ // Handle different values for --think+ switch thinkStr {+ case "", "true":+ // --think or --think=true+ opts.Think = &api.ThinkValue{Value: true}+ case "false":+ opts.Think = &api.ThinkValue{Value: false}+ case "high", "medium", "low":+ opts.Think = &api.ThinkValue{Value: thinkStr}+ default:+ return fmt.Errorf("invalid value for --think: %q (must be true, false, high, medium, or low)", thinkStr)+ }} else {opts.Think = nil}@@ -977,7 +989,7 @@ type runOptions struct {Options map[string]anyMultiModal boolKeepAlive *api.Duration- Think *bool+ Think *api.ThinkValueHideThinking bool}@@ -1017,10 +1029,11 @@ func displayResponse(content string, wordWrap bool, state *displayResponseState)}switch ch {- case ' ':+ case ' ', '\t':state.wordBuffer = ""- case '\n':+ case '\n', '\r':state.lineLength = 0+ state.wordBuffer = ""default:state.wordBuffer += string(ch)}@@ -1078,6 +1091,7 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {}()var state *displayResponseState = &displayResponseState{}+ var thinkingContent strings.Buildervar latest api.ChatResponsevar fullResponse strings.Buildervar thinkTagOpened bool = false@@ -1097,14 +1111,21 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {if !thinkTagOpened {fmt.Print(thinkingOutputOpeningText(false))thinkTagOpened = true+ thinkTagClosed = false}+ thinkingContent.WriteString(response.Message.Thinking)displayResponse(response.Message.Thinking, opts.WordWrap, state)}content := response.Message.Content- if thinkTagOpened && !thinkTagClosed && content != "" {+ if thinkTagOpened && !thinkTagClosed && (content != "" || len(response.Message.ToolCalls) > 0) {+ if !strings.HasSuffix(thinkingContent.String(), "\n") {+ fmt.Println()+ }fmt.Print(thinkingOutputClosingText(false))+ thinkTagOpened = falsethinkTagClosed = true+ state = &displayResponseState{}}// purposefully not putting thinking blocks in the response, which would// only be needed if we later added tool calling to the cli (they get@@ -1112,6 +1133,13 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {// about to finish some tool calls)fullResponse.WriteString(content)+ if response.Message.ToolCalls != nil {+ toolCalls := response.Message.ToolCalls+ if len(toolCalls) > 0 {+ fmt.Print(renderToolCalls(toolCalls, false))+ }+ }+displayResponse(content, opts.WordWrap, state)return nil@@ -1196,6 +1224,7 @@ func generate(cmd *cobra.Command, opts runOptions) error {}()var state *displayResponseState = &displayResponseState{}+ var thinkingContent strings.Buildervar thinkTagOpened bool = falsevar thinkTagClosed bool = false@@ -1213,17 +1242,31 @@ func generate(cmd *cobra.Command, opts runOptions) error {if !thinkTagOpened {fmt.Print(thinkingOutputOpeningText(plainText))thinkTagOpened = true+ thinkTagClosed = false}+ thinkingContent.WriteString(response.Thinking)displayResponse(response.Thinking, opts.WordWrap, state)}- if thinkTagOpened && !thinkTagClosed && content != "" {+ if thinkTagOpened && !thinkTagClosed && (content != "" || len(response.ToolCalls) > 0) {+ if !strings.HasSuffix(thinkingContent.String(), "\n") {+ fmt.Println()+ }fmt.Print(thinkingOutputClosingText(plainText))+ thinkTagOpened = falsethinkTagClosed = true+ state = &displayResponseState{}}displayResponse(content, opts.WordWrap, state)+ if response.ToolCalls != nil {+ toolCalls := response.ToolCalls+ if len(toolCalls) > 0 {+ fmt.Print(renderToolCalls(toolCalls, plainText))+ }+ }+return nil}@@ -1463,7 +1506,8 @@ func NewCLI() *cobra.Command {runCmd.Flags().Bool("insecure", false, "Use an insecure registry")runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")runCmd.Flags().String("format", "", "Response format (e.g. json)")- runCmd.Flags().Bool("think", false, "Whether to use thinking mode for supported models")+ runCmd.Flags().String("think", "", "Enable thinking mode: true/false or high/medium/low for supported models")+ runCmd.Flags().Lookup("think").NoOptDefVal = "true"runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")stopCmd := &cobra.Command{@@ -1613,7 +1657,7 @@ func NewCLI() *cobra.Command {// to false).//// If capabilities are not provided, we fetch them from the server.-func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicitlySetByUser bool) (*bool, error) {+func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicitlySetByUser bool) (*api.ThinkValue, error) {if explicitlySetByUser {return runOpts.Think, nil}@@ -1640,9 +1684,34 @@ func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicit}if thinkingSupported {- thinking := true- return &thinking, nil+ return &api.ThinkValue{Value: true}, nil}return nil, nil}++func renderToolCalls(toolCalls []api.ToolCall, plainText bool) string {+ out := ""+ formatExplanation := ""+ formatValues := ""+ if !plainText {+ formatExplanation = readline.ColorGrey + readline.ColorBold+ formatValues = readline.ColorDefault+ out += formatExplanation+ }+ for i, toolCall := range toolCalls {+ argsAsJSON, err := json.Marshal(toolCall.Function.Arguments)+ if err != nil {+ return ""+ }+ if i > 0 {+ out += "\n"+ }+ // all tool calls are unexpected since we don't currently support registering any in the CLI+ out += fmt.Sprintf(" Model called a non-existent function '%s()' with arguments: %s", formatValues+toolCall.Function.Name+formatExplanation, formatValues+string(argsAsJSON)+formatExplanation)+ }+ if !plainText {+ out += readline.ColorDefault+ }+ return out+}diff --git a/cmd/interactive.go b/cmd/interactive.goindex 08ab4947b..e290d84ce 100644--- a/cmd/interactive.go+++ b/cmd/interactive.go@@ -272,16 +272,29 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {}fmt.Println("Set 'quiet' mode.")case "think":- think := true- opts.Think = &think+ thinkValue := api.ThinkValue{Value: true}+ var maybeLevel string+ if len(args) > 2 {+ maybeLevel = args[2]+ }+ if maybeLevel != "" {+ // TODO(drifkin): validate the level, could be model dependent+ // though... It will also be validated on the server once a call is+ // made.+ thinkValue.Value = maybeLevel+ }+ opts.Think = &thinkValuethinkExplicitlySet = trueif client, err := api.ClientFromEnvironment(); err == nil {ensureThinkingSupport(cmd.Context(), client, opts.Model)}- fmt.Println("Set 'think' mode.")+ if maybeLevel != "" {+ fmt.Printf("Set 'think' mode to '%s'.\n", maybeLevel)+ } else {+ fmt.Println("Set 'think' mode.")+ }case "nothink":- think := false- opts.Think = &think+ opts.Think = &api.ThinkValue{Value: false}thinkExplicitlySet = trueif client, err := api.ClientFromEnvironment(); err == nil {ensureThinkingSupport(cmd.Context(), client, opts.Model)@@ -478,7 +491,8 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {assistant, err := chat(cmd, opts)if err != nil {- if strings.Contains(err.Error(), "does not support thinking") {+ if strings.Contains(err.Error(), "does not support thinking") ||+ strings.Contains(err.Error(), "invalid think value") {fmt.Printf("error: %v\n", err)sb.Reset()continuediff --git a/convert/convert.go b/convert/convert.goindex 63b3bf661..bed59a575 100644--- a/convert/convert.go+++ b/convert/convert.go@@ -202,6 +202,8 @@ func ConvertModel(fsys fs.FS, f *os.File) error {conv = &bertModel{}case "CohereForCausalLM":conv = &commandrModel{}+ case "GptOssForCausalLM":+ conv = &gptossModel{}default:return fmt.Errorf("unsupported architecture %q", p.Architectures[0])}diff --git a/convert/convert_gptoss.go b/convert/convert_gptoss.gonew file mode 100644index 000000000..bd362169b--- /dev/null+++ b/convert/convert_gptoss.go@@ -0,0 +1,178 @@+package convert++import (+ "bytes"+ "cmp"+ "encoding/binary"+ "io"+ "slices"+ "strings"++ "github.com/ollama/ollama/fs/ggml"+ "github.com/pdevine/tensor"+ "github.com/pdevine/tensor/native"+)++type gptossModel struct {+ ModelParameters+ HiddenLayers uint32 `json:"num_hidden_layers"`+ HiddenSize uint32 `json:"hidden_size"`+ IntermediateSize uint32 `json:"intermediate_size"`+ AttentionHeads uint32 `json:"num_attention_heads"`+ KeyValueHeads uint32 `json:"num_key_value_heads"`+ HeadDim uint32 `json:"head_dim"`+ Experts uint32 `json:"num_experts"`+ ExpertsPerToken uint32 `json:"experts_per_token"`+ RMSNormEpsilon float32 `json:"rms_norm_eps"`+ InitialContextLength uint32 `json:"initial_context_length"`+ RopeTheta float32 `json:"rope_theta"`+ RopeScalingFactor float32 `json:"rope_scaling_factor"`+ SlidingWindow uint32 `json:"sliding_window"`+}++var _ ModelConverter = (*gptossModel)(nil)++func (m *gptossModel) KV(t *Tokenizer) ggml.KV {+ kv := m.ModelParameters.KV(t)+ kv["general.architecture"] = "gptoss"+ kv["general.file_type"] = uint32(4)+ kv["gptoss.context_length"] = uint32(m.RopeScalingFactor * float32(m.InitialContextLength))+ kv["gptoss.block_count"] = m.HiddenLayers+ kv["gptoss.embedding_length"] = m.HiddenSize+ kv["gptoss.feed_forward_length"] = m.IntermediateSize+ kv["gptoss.expert_count"] = m.Experts+ kv["gptoss.expert_used_count"] = m.ExpertsPerToken+ kv["gptoss.attention.head_count"] = m.AttentionHeads+ kv["gptoss.attention.head_count_kv"] = m.KeyValueHeads+ kv["gptoss.attention.key_length"] = m.HeadDim+ kv["gptoss.attention.value_length"] = m.HeadDim+ kv["gptoss.attention.layer_norm_rms_epsilon"] = cmp.Or(m.RMSNormEpsilon, 1e-5)+ kv["gptoss.attention.sliding_window"] = m.SlidingWindow+ kv["gptoss.rope.freq_base"] = m.RopeTheta+ kv["gptoss.rope.scaling.factor"] = m.RopeScalingFactor+ kv["gptoss.rope.scaling.original_context_length"] = m.InitialContextLength+ kv["tokenizer.ggml.bos_token_id"] = uint32(199998) // <|startoftext|>+ kv["tokenizer.ggml.add_bos_token"] = false+ kv["tokenizer.ggml.eos_token_id"] = uint32(199999) // <|endoftext|>+ kv["tokenizer.ggml.eos_token_ids"] = []int32{+ 199999, /* <|endoftext|> */+ 200002, /* <|return|> */+ 200012, /* <|call|> */+ }+ kv["tokenizer.ggml.add_eos_token"] = false+ return kv+}++func (m *gptossModel) Tensors(ts []Tensor) []*ggml.Tensor {+ var out []*ggml.Tensor+ mxfp4s := make(map[string]*mxfp4)+ for _, t := range ts {+ if strings.HasSuffix(t.Name(), ".blocks") || strings.HasSuffix(t.Name(), ".scales") {+ dot := strings.LastIndex(t.Name(), ".")+ name, suffix := t.Name()[:dot], t.Name()[dot+1:]+ if _, ok := mxfp4s[name]; !ok {+ mxfp4s[name] = &mxfp4{}+ }++ switch suffix {+ case "blocks":+ mxfp4s[name].blocks = t+ case "scales":+ mxfp4s[name].scales = t+ }+ } else {+ out = append(out, &ggml.Tensor{+ Name: t.Name(),+ Kind: t.Kind(),+ Shape: t.Shape(),+ WriterTo: t,+ })+ }+ }++ for name, mxfp4 := range mxfp4s {+ dims := mxfp4.blocks.Shape()+ out = append(out, &ggml.Tensor{+ Name: name,+ Kind: uint32(ggml.TensorTypeMXFP4),+ Shape: []uint64{dims[0], dims[1], dims[2] * dims[3] * 2},+ WriterTo: mxfp4,+ })+ }++ return out+}++func (m *gptossModel) Replacements() []string {+ return []string{+ // noop replacements so other replacements will not be applied+ ".blocks", ".blocks",+ ".scales", ".scales",+ // real replacements+ "block", "blk",+ "attn.norm", "attn_norm",+ "attn.qkv", "attn_qkv",+ "attn.sinks", "attn_sinks",+ "attn.out", "attn_out",+ "mlp.norm", "ffn_norm",+ "mlp.gate", "ffn_gate_inp",+ "mlp.mlp1_", "ffn_gate_up_exps.",+ "mlp.mlp2_", "ffn_down_exps.",+ "embedding", "token_embd",+ "norm", "output_norm",+ "unembedding", "output",+ "scale", "weight",+ }+}++type mxfp4 struct {+ blocks, scales Tensor+}++func (m *mxfp4) WriteTo(w io.Writer) (int64, error) {+ var b bytes.Buffer+ if _, err := m.blocks.WriteTo(&b); err != nil {+ return 0, err+ }++ blocksDims := make([]int, len(m.blocks.Shape()))+ for i, d := range m.blocks.Shape() {+ blocksDims[i] = int(d)+ }++ var blocks tensor.Tensor = tensor.New(tensor.WithShape(blocksDims...), tensor.WithBacking(b.Bytes()))++ var s bytes.Buffer+ if _, err := m.scales.WriteTo(&s); err != nil {+ return 0, err+ }++ scalesDims := slices.Repeat([]int{1}, len(m.blocks.Shape()))+ for i, d := range m.scales.Shape() {+ scalesDims[i] = int(d)+ }++ var scales tensor.Tensor = tensor.New(tensor.WithShape(scalesDims...), tensor.WithBacking(s.Bytes()))++ out, err := tensor.Concat(3, scales, blocks)+ if err != nil {+ return 0, err+ }++ out = tensor.Materialize(out)++ if err := out.Reshape(out.Shape().TotalSize()); err != nil {+ return 0, err+ }++ u8s, err := native.VectorU8(out.(*tensor.Dense))+ if err != nil {+ return 0, err+ }++ if err := binary.Write(w, binary.LittleEndian, u8s); err != nil {+ return 0, err+ }++ return 0, nil+}diff --git a/convert/reader.go b/convert/reader.goindex 07d12f0dd..367e91a29 100644--- a/convert/reader.go+++ b/convert/reader.go@@ -31,8 +31,10 @@ func (t tensorBase) Shape() []uint64 {}const (- tensorKindF32 uint32 = iota- tensorKindF16+ tensorKindFP32 uint32 = iota+ tensorKindFP16+ tensorKindMXFP4 = 4+ tensorKindBF16 = 30)func (t tensorBase) Kind() uint32 {@@ -43,16 +45,16 @@ func (t tensorBase) Kind() uint32 {t.name == "v.pre_tile_position_embd.weight" ||t.name == "v.post_tile_position_embd.weight" {// these tensors are always F32- return 0+ return tensorKindFP32}switch len(t.shape) {case 0:panic("invalid tensor shape")case 1:- return tensorKindF32+ return tensorKindFP32default:- return tensorKindF16+ return tensorKindFP16}}diff --git a/convert/reader_safetensors.go b/convert/reader_safetensors.goindex f182a656c..63f31631d 100644--- a/convert/reader_safetensors.go+++ b/convert/reader_safetensors.go@@ -93,6 +93,15 @@ type safetensor struct {*tensorBase}+func (st safetensor) Kind() uint32 {+ kind := st.tensorBase.Kind()+ if st.dtype == "BF16" && kind != tensorKindFP32 {+ kind = tensorKindBF16+ }++ return kind+}+func (st safetensor) Clone() Tensor {return &safetensor{fs: st.fs,@@ -150,6 +159,9 @@ func (st safetensor) WriteTo(w io.Writer) (int64, error) {}f32s = bfloat16.DecodeFloat32(u8s)+ case "U8":+ // U8 tensors do not support repacking or type conversion.+ return io.CopyN(w, f, st.size)default:return 0, fmt.Errorf("unknown data type: %s", st.dtype)}@@ -162,15 +174,18 @@ func (st safetensor) WriteTo(w io.Writer) (int64, error) {}switch st.Kind() {- case tensorKindF32:+ case tensorKindFP32:return 0, binary.Write(w, binary.LittleEndian, f32s)- case tensorKindF16:+ case tensorKindFP16:f16s := make([]uint16, len(f32s))for i := range f32s {f16s[i] = float16.Fromfloat32(f32s[i]).Bits()}return 0, binary.Write(w, binary.LittleEndian, f16s)+ case tensorKindBF16:+ u8s := bfloat16.EncodeFloat32(f32s)+ return 0, binary.Write(w, binary.LittleEndian, u8s)default:return 0, fmt.Errorf("unknown storage type: %d", st.Kind())}diff --git a/convert/tensor_test.go b/convert/tensor_test.goindex 0b2db5baa..3a34bbff6 100644--- a/convert/tensor_test.go+++ b/convert/tensor_test.go@@ -72,236 +72,787 @@ func mul(shape []uint64) int {}func TestSplitDim(t *testing.T) {- r := fakeTensor{- name: "a.b",- shape: []uint64{3, 4},- data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},- }-- t.Run("no split", func(t *testing.T) {- for tt := range splitDim(&r, 0, split{Replacer: strings.NewReplacer("a", "x")}) {- if tt.Name != "x.b" {- t.Fatalf("expected name 'x', got '%s'", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{3, 4}) {- t.Fatalf("expected shape [3, 4], got %v", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) {- t.Fatalf("expected data [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], got %v", f32s)- }- }- })-- t.Run("even split", func(t *testing.T) {- next, stop := iter.Pull(splitDim(&r, 1,- split{Replacer: strings.NewReplacer("a", "x")},- split{Replacer: strings.NewReplacer("b", "y")},- ))- defer stop()-- {- tt, ok := next()- if !ok {- t.Fatal("expected at least one split")- }-- if tt.Name != "x.b" {- t.Fatal("expected name 'x.b', got", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{3, 2}) {- t.Fatal("expected shape [3, 2], got", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{0, 1, 4, 5, 8, 9}) {- t.Fatal("expected data [0, 1, 4, 5, 8, 9], got", f32s)- }- }-- {- tt, ok := next()- if !ok {- t.Fatal("expected at least one split")- }-- if tt.Name != "a.y" {- t.Fatal("expected name 'a.y', got", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{3, 2}) {- t.Fatal("expected shape [3, 2], got", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{2, 3, 6, 7, 10, 11}) {- t.Fatal("expected data [2, 3, 6, 7, 10, 11], got", f32s)- }- }- })-- t.Run("uneven split", func(t *testing.T) {- next, stop := iter.Pull(splitDim(&r, 0,- split{Replacer: strings.NewReplacer("a", "x"), dim: 2},- split{Replacer: strings.NewReplacer("b", "y"), dim: 1},- ))- defer stop()-- {- tt, ok := next()- if !ok {- t.Fatal("expected at least one split")- }-- if tt.Name != "x.b" {- t.Fatal("expected name 'x.b', got", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{2, 4}) {- t.Fatal("expected shape [2, 4], got", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7}) {- t.Fatal("expected data [0, 1, 2, 3, 4, 5, 6, 7], got", f32s)- }+ t.Run("2d", func(t *testing.T) {+ r := fakeTensor{+ name: "a.b",+ shape: []uint64{3, 4},+ data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},}- {- tt, ok := next()- if !ok {- t.Fatal("expected at least one split")- }-- if tt.Name != "a.y" {- t.Fatal("expected name 'a.y', got", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{1, 4}) {- t.Fatal("expected shape [1, 4], got", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{8, 9, 10, 11}) {- t.Fatal("expected data [8, 9, 10, 11], got", f32s)- }- }+ t.Run("no split", func(t *testing.T) {+ for tt := range splitDim(&r, 0, split{Replacer: strings.NewReplacer("a", "x")}) {+ if tt.Name != "x.b" {+ t.Fatalf("expected name 'x', got '%s'", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 4}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("even split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 1,+ split{Replacer: strings.NewReplacer("a", "x")},+ split{Replacer: strings.NewReplacer("b", "y")},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 4, 5, 8, 9}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'a.y', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{2, 3, 6, 7, 10, 11}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("uneven split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 0,+ split{Replacer: strings.NewReplacer("a", "x"), dim: 2},+ split{Replacer: strings.NewReplacer("b", "y"), dim: 1},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{2, 4}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'a.y', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{8, 9, 10, 11}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("three way split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 0,+ split{Replacer: strings.NewReplacer("a", "x"), dim: 1},+ split{Replacer: strings.NewReplacer("b", "y"), dim: 1},+ split{Replacer: strings.NewReplacer("b", "z"), dim: 1},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{4, 5, 6, 7}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.z" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{8, 9, 10, 11}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("uneven three way split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 1,+ split{Replacer: strings.NewReplacer("a", "x"), dim: 2},+ split{Replacer: strings.NewReplacer("b", "y"), dim: 1},+ split{Replacer: strings.NewReplacer("b", "z"), dim: 1},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 4, 5, 8, 9}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 1}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{2, 6, 10}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.z" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 1}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{3, 7, 11}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("split with transpose", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 1,+ split{Replacer: strings.NewReplacer("a", "x")},+ split{Replacer: strings.NewReplacer("b", "y"), fn: func(tt tensor.Tensor) (tensor.Tensor, error) {+ return tensor.Transpose(tt, 1, 0)+ }},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 4, 5, 8, 9}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'a.y', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{2, 6, 10, 3, 7, 11}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })})-- t.Run("split with transpose", func(t *testing.T) {- next, stop := iter.Pull(splitDim(&r, 1,- split{Replacer: strings.NewReplacer("a", "x")},- split{Replacer: strings.NewReplacer("b", "y"), fn: func(tt tensor.Tensor) (tensor.Tensor, error) {- return tensor.Transpose(tt, 1, 0)- }},- ))- defer stop()-- {- tt, ok := next()- if !ok {- t.Fatal("expected at least one split")- }-- if tt.Name != "x.b" {- t.Fatal("expected name 'x.b', got", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{3, 2}) {- t.Fatal("expected shape [3, 2], got", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{0, 1, 4, 5, 8, 9}) {- t.Fatal("expected data [0, 1, 4, 5, 8, 9], got", f32s)- }+ t.Run("3d", func(t *testing.T) {+ r := fakeTensor{+ name: "a.b",+ shape: []uint64{3, 4, 2},+ data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},}- {- tt, ok := next()- if !ok {- t.Fatal("expected at least one split")- }-- if tt.Name != "a.y" {- t.Fatal("expected name 'a.y', got", tt.Name)- }-- if !slices.Equal(tt.Shape, []uint64{3, 2}) {- t.Fatal("expected shape [3, 2], got", tt.Shape)- }-- var b bytes.Buffer- if _, err := tt.WriteTo(&b); err != nil {- t.Fatal(err)- }-- f32s := make([]float32, mul(tt.Shape))- if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {- t.Fatal(err)- }-- if !slices.Equal(f32s, []float32{2, 6, 10, 3, 7, 11}) {- t.Fatal("expected data [2, 6, 10, 3, 7, 11], got", f32s)- }- }+ t.Run("no split", func(t *testing.T) {+ for tt := range splitDim(&r, 0, split{Replacer: strings.NewReplacer("a", "x")}) {+ if tt.Name != "x.b" {+ t.Fatalf("expected name 'x', got '%s'", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 4, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("even split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 1,+ split{Replacer: strings.NewReplacer("a", "x")},+ split{Replacer: strings.NewReplacer("b", "y")},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'a.y', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("uneven split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 0,+ split{Replacer: strings.NewReplacer("a", "x"), dim: 2},+ split{Replacer: strings.NewReplacer("b", "y"), dim: 1},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{2, 4, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'a.y', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{16, 17, 18, 19, 20, 21, 22, 23}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("three way split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 0,+ split{Replacer: strings.NewReplacer("a", "x"), dim: 1},+ split{Replacer: strings.NewReplacer("b", "y"), dim: 1},+ split{Replacer: strings.NewReplacer("b", "z"), dim: 1},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 4, 5, 6, 7}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{8, 9, 10, 11, 12, 13, 14, 15}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.z" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{1, 4, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{16, 17, 18, 19, 20, 21, 22, 23}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })++ t.Run("uneven three way split", func(t *testing.T) {+ next, stop := iter.Pull(splitDim(&r, 1,+ split{Replacer: strings.NewReplacer("a", "x"), dim: 2},+ split{Replacer: strings.NewReplacer("b", "y"), dim: 1},+ split{Replacer: strings.NewReplacer("b", "z"), dim: 1},+ ))+ defer stop()++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "x.b" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 2, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.y" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 1, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{4, 5, 12, 13, 20, 21}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }++ {+ tt, ok := next()+ if !ok {+ t.Fatal("expected at least one split")+ }++ if tt.Name != "a.z" {+ t.Fatal("expected name 'x.b', got", tt.Name)+ }++ if diff := cmp.Diff(tt.Shape, []uint64{3, 1, 2}); diff != "" {+ t.Errorf("unexpected shape (-want +got):\n%s", diff)+ }++ var b bytes.Buffer+ if _, err := tt.WriteTo(&b); err != nil {+ t.Fatal(err)+ }++ f32s := make([]float32, mul(tt.Shape))+ if err := binary.Read(&b, binary.LittleEndian, &f32s); err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff(f32s, []float32{6, 7, 14, 15, 22, 23}); diff != "" {+ t.Errorf("unexpected data (-want +got):\n%s", diff)+ }+ }+ })})}diff --git a/fs/ggml/ggml.go b/fs/ggml/ggml.goindex 32f459a3a..afb90720f 100644--- a/fs/ggml/ggml.go+++ b/fs/ggml/ggml.go@@ -1,6 +1,7 @@package ggmlimport (+ "cmp""encoding/binary""errors""fmt"@@ -179,6 +180,7 @@ func (kv KV) OllamaEngineRequired() bool {"llama4","mllama","qwen25vl",+ "gptoss",}, kv.Architecture())}@@ -280,7 +282,7 @@ func (t Tensor) block() (n int) {}func (t Tensor) blockSize() uint64 {- return (TensorType)(t.Kind).BlockSize()+ return TensorType(t.Kind).BlockSize()}func (t TensorType) BlockSize() uint64 {@@ -298,6 +300,7 @@ func (t TensorType) BlockSize() uint64 {case2, // Q4_03, // Q4_1+ 4, // MXFP46, // Q5_07, // Q5_18, // Q8_0@@ -325,6 +328,8 @@ func (t TensorType) TypeSize() uint64 {return 2 + blockSize/2case TensorTypeQ4_1:return 2 + 2 + blockSize/2+ case TensorTypeMXFP4:+ return 1 + blockSize/2case TensorTypeQ5_0:return 2 + 4 + blockSize/2case TensorTypeQ5_1:@@ -487,9 +492,11 @@ func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType strilayers := f.Tensors().GroupLayers()bytesPerElement := kvCacheBytesPerElement(kvCacheType)+ var kvTotal uint64kv = make([]uint64, f.KV().BlockCount())for i := range kv {kv[i] = uint64(float64(context*(embeddingHeadsK+embeddingHeadsV)*headsKV) * bytesPerElement)+ kvTotal += kv[i]}switch f.KV().Architecture() {@@ -658,6 +665,18 @@ func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType stri4*qkvBias.Shape[0],)}+ case "gptoss":+ kv = make([]uint64, f.KV().BlockCount())+ for i := range kv {+ kv[i] = uint64(float64((embeddingHeadsK+embeddingHeadsV)*headsKV) * bytesPerElement)+ if i%2 == 0 {+ kv[i] *= (uint64(numParallel)*4096 + batch)+ } else {+ kv[i] *= context+ }+ }+ fullOffload = 4 * f.KV().HeadCountMax() / cmp.Or(f.KV().HeadCountKVMin(), 1) * kvTotal / 6+ partialOffload = 2 * fullOffload}returndiff --git a/fs/ggml/type.go b/fs/ggml/type.goindex 4d3d5bcad..3e5deb87b 100644--- a/fs/ggml/type.go+++ b/fs/ggml/type.go@@ -14,9 +14,9 @@ const (FileTypeF16fileTypeQ4_0fileTypeQ4_1- fileTypeQ4_1_F16 // unused by GGML- fileTypeQ4_2 // unused by GGML- fileTypeQ4_3 // unused by GGML+ fileTypeMXFP4 // originally fileTypeQ4_1_F16 // unused by GGML+ fileTypeQ4_2 // unused by GGML+ fileTypeQ4_3 // unused by GGMLFileTypeQ8_0fileTypeQ5_0fileTypeQ5_1@@ -97,6 +97,8 @@ func (t FileType) String() string {return "Q4_0"case fileTypeQ4_1:return "Q4_1"+ case fileTypeMXFP4:+ return "MXFP4"case FileTypeQ8_0:return "Q8_0"case fileTypeQ5_0:@@ -144,6 +146,8 @@ func (ftype FileType) ToTensorType() TensorType {return TensorTypeQ4_0case fileTypeQ4_1:return TensorTypeQ4_1+ case fileTypeMXFP4:+ return TensorTypeMXFP4 // Formerly unused tensorTypeQ4_2case FileTypeQ8_0:return TensorTypeQ8_0case fileTypeQ5_0:@@ -187,8 +191,8 @@ const (TensorTypeF16TensorTypeQ4_0TensorTypeQ4_1- tensorTypeQ4_2 // unused by GGML- tensorTypeQ4_3 // unused by GGML+ TensorTypeMXFP4 // Formerly unused tensorTypeQ4_2+ tensorTypeQ4_3 // unused by GGMLTensorTypeQ5_0TensorTypeQ5_1TensorTypeQ8_0@@ -260,6 +264,8 @@ func ParseTensorType(s string) (TensorType, error) {return TensorTypeF64, nilcase "BF16":return TensorTypeBF16, nil+ case "MXFP4":+ return TensorTypeMXFP4, nildefault:return 0, fmt.Errorf("unsupported quantization type %s", s)}@@ -312,6 +318,8 @@ func (t TensorType) String() string {return "F64"case TensorTypeBF16:return "BF16"+ case TensorTypeMXFP4:+ return "MXFP4"default:return "unknown"}diff --git a/llama/patches/0019-metal-add-mean-kernel-14267.patch b/llama/patches/0019-metal-add-mean-kernel-14267.patchindex f20e854b2..e65aeb7b4 100644--- a/llama/patches/0019-metal-add-mean-kernel-14267.patch+++ b/llama/patches/0019-metal-add-mean-kernel-14267.patch@@ -19,7 +19,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.mindex a9eeebc6..110c9ece 100644--- a/ggml/src/ggml-metal/ggml-metal.m+++ b/ggml/src/ggml-metal/ggml-metal.m-@@ -489,6 +489,7 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte+@@ -489,6 +489,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_COS,GGML_METAL_KERNEL_TYPE_NEG,GGML_METAL_KERNEL_TYPE_SUM_ROWS,@@ -27,7 +27,7 @@ index a9eeebc6..110c9ece 100644GGML_METAL_KERNEL_TYPE_POOL_2D_AVG_F32,GGML_METAL_KERNEL_TYPE_POOL_2D_MAX_F32,GGML_METAL_KERNEL_TYPE_ARGMAX,-@@ -1436,6 +1437,7 @@ @implementation GGMLMetalClass+@@ -1436,6 +1437,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t deGGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_COS, cos, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NEG, neg, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);diff --git a/llama/patches/0022-BF16-macos-version-guard.patch b/llama/patches/0022-BF16-macos-version-guard.patchindex 68aac0bb0..88e4f7cb0 100644--- a/llama/patches/0022-BF16-macos-version-guard.patch+++ b/llama/patches/0022-BF16-macos-version-guard.patch@@ -12,7 +12,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.mindex 110c9ece..ab46f6e3 100644--- a/ggml/src/ggml-metal/ggml-metal.m+++ b/ggml/src/ggml-metal/ggml-metal.m-@@ -89,7 +89,11 @@+@@ -89,7 +89,11 @@ static id<MTLDevice> ggml_backend_metal_device_acq(struct ggml_backend_metal_devctx->has_bfloat |= [ctx->mtl_device supportsFamily:MTLGPUFamilyApple6];#if defined(GGML_METAL_USE_BF16)diff --git a/llama/patches/0023-MXFP4.patch b/llama/patches/0023-MXFP4.patchnew file mode 100644index 000000000..2beb1518d--- /dev/null+++ b/llama/patches/0023-MXFP4.patch@@ -0,0 +1,1293 @@+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001+From: Daniel Hiltgen <[email protected]>+Date: Mon, 21 Jul 2025 12:06:13 -0700+Subject: [PATCH] MXFP4++Partial implementation of MXFP4 tensor type+---+ ggml/include/ggml.h | 2 +-+ ggml/src/ggml-common.h | 7 ++ ggml/src/ggml-cpu/ggml-cpu-quants.h | 2 ++ ggml/src/ggml-cpu/ggml-cpu.c | 5 ++ ggml/src/ggml-cpu/ops.cpp | 1 ++ ggml/src/ggml-cpu/vec.cpp | 90 +++++++++ ggml/src/ggml-cpu/vec.h | 2 ++ ggml/src/ggml-cuda/convert.cu | 80 ++++++++ ggml/src/ggml-cuda/ggml-cuda.cu | 16 +-+ ggml/src/ggml-cuda/mmvmxfp4.cu | 307 +++++++++++++++++++++++++++ ggml/src/ggml-cuda/mmvmxfp4.cuh | 9 ++ ggml/src/ggml-metal/ggml-metal-impl.h | 3 ++ ggml/src/ggml-metal/ggml-metal.m | 25 ++-+ ggml/src/ggml-metal/ggml-metal.metal | 173 ++++++++++++++-+ ggml/src/ggml-quants.c | 142 +++++++++++-+ ggml/src/ggml-quants.h | 6 ++ ggml/src/ggml.c | 13 +-+ 17 files changed, 868 insertions(+), 15 deletions(-)+ create mode 100644 ggml/src/ggml-cuda/mmvmxfp4.cu+ create mode 100644 ggml/src/ggml-cuda/mmvmxfp4.cuh++diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h+index e91dedf1..873baa24 100644+--- a/ggml/include/ggml.h++++ b/ggml/include/ggml.h+@@ -353,7 +353,7 @@ extern "C" {+ GGML_TYPE_F16 = 1,+ GGML_TYPE_Q4_0 = 2,+ GGML_TYPE_Q4_1 = 3,+- // GGML_TYPE_Q4_2 = 4, support has been removed++ GGML_TYPE_MXFP4 = 4, // Formerly removed type GGML_TYPE_Q4_2+ // GGML_TYPE_Q4_3 = 5, support has been removed+ GGML_TYPE_Q5_0 = 6,+ GGML_TYPE_Q5_1 = 7,+diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h+index 086c822d..e0d71451 100644+--- a/ggml/src/ggml-common.h++++ b/ggml/src/ggml-common.h+@@ -417,6 +417,13 @@ typedef struct {+ } block_iq4_xs;+ static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");+++#define MXFP4 32++typedef struct {++ uint8_t d; // scale E8M0 float++ uint8_t qs[MXFP4 / 2]; // (32) 4 bit elements E2M1 float++} block_mxfp4;++static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + MXFP4/2, "wrong mxfp4 block size/padding");+++ #endif // GGML_COMMON_DECL+ #endif // GGML_COMMON_DECL++diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.h b/ggml/src/ggml-cpu/ggml-cpu-quants.h+index e33d9d47..6a25d062 100644+--- a/ggml/src/ggml-cpu/ggml-cpu-quants.h++++ b/ggml/src/ggml-cpu/ggml-cpu-quants.h+@@ -58,6 +58,8 @@ void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const+ void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+++void ggml_vec_dot_mxfp4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);+++ #ifdef __cplusplus+ }+ #endif+diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c+index 2462d2b8..bff9c426 100644+--- a/ggml/src/ggml-cpu/ggml-cpu.c++++ b/ggml/src/ggml-cpu/ggml-cpu.c+@@ -362,6 +362,11 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {+ .vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,+ },++ [GGML_TYPE_MXFP4] = {++ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_mxfp4,++ .vec_dot_type = GGML_TYPE_F32,++ .nrows = 1,++ },+ };++ const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {+diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp+index 654e2f28..be0aa683 100644+--- a/ggml/src/ggml-cpu/ops.cpp++++ b/ggml/src/ggml-cpu/ops.cpp+@@ -4965,6 +4965,7 @@ void ggml_compute_forward_clamp(+ case GGML_TYPE_I32:+ case GGML_TYPE_I64:+ case GGML_TYPE_F64:++ case GGML_TYPE_MXFP4:+ case GGML_TYPE_COUNT:+ {+ GGML_ABORT("fatal error");+diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp+index 02d40618..ec3ec9b1 100644+--- a/ggml/src/ggml-cpu/vec.cpp++++ b/ggml/src/ggml-cpu/vec.cpp+@@ -250,3 +250,93 @@ ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, fl+ }+ return sum = (ggml_float)logf(sum);+ }++++#define MXFP4 32++typedef struct {++ uint8_t d; // scale E8M0 float++ uint8_t qs[MXFP4 / 2]; // (32) 4 bit elements E2M1 float++} block_mxfp4;++static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + MXFP4/2, "wrong mxfp4 block size/padding");++#define MXFP4_VALS {0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, 0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0}++++void ggml_vec_dot_mxfp4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {++ assert(nrc == 1);++ GGML_UNUSED(nrc);++ GGML_UNUSED(bx);++ GGML_UNUSED(by);++ GGML_UNUSED(bs);++ ggml_float mxfp4_table[] = MXFP4_VALS;++++#if defined(GGML_SIMD)++ float sumf = 0.0f;++ const int np = (n & ~(GGML_F32_STEP - 1));++ const block_mxfp4 * GGML_RESTRICT xx = (const block_mxfp4 *) vx;++ GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };++++ GGML_F32_VEC scalev;++ GGML_F32_VEC ax[GGML_F32_ARR];++ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) { // ARM: +16 AVX512: +64++ for (int j = 0; j < GGML_F32_ARR; j++) { // ARM: 0 .. 4 AVX512: 0 .. 4++ // convert GGML_F32_ARR X elements++ const int ib = (i + j*GGML_F32_EPR) / MXFP4;++ const block_mxfp4 * GGML_RESTRICT x = &xx[ib];++ union {++ uint32_t as_bits;++ float as_value;++ } scale;++ scale.as_bits = (((uint32_t)x->d) << 23);++ scalev = GGML_F32_VEC_SET1(scale.as_value);++ float xf[GGML_F32_EPR]= {0.f};++ assert(((i+j*GGML_F32_EPR) % MXFP4)+GGML_F32_ARR < MXFP4 && "block overrun");++ for (int qi = 0; qi < GGML_F32_EPR/2 ; ++qi) {++ xf[qi*2] = mxfp4_table[(x->qs[((i+j*GGML_F32_EPR)%MXFP4)/2+qi] & 0xf)];++ xf[qi*2+1] = mxfp4_table[(x->qs[((i+j*GGML_F32_EPR)%MXFP4)/2+qi] & 0xf0) >> 4];++ }++++ ax[j] = GGML_F32_VEC_MUL(GGML_F32_VEC_LOAD(xf), scalev);++ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);++ sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);++ }++ }++ GGML_F32_VEC_REDUCE(sumf, sum);++++ // leftovers++ for (int i = np; i < n; i+=2) {++ const int ib = i / MXFP4;++ const block_mxfp4 * GGML_RESTRICT x = &xx[ib];++ union {++ uint32_t as_bits;++ float as_value;++ } scale;++ scale.as_bits = (((uint32_t)x->d) << 23);++ sumf += y[i] * scale.as_value * mxfp4_table[(x->qs[(i%MXFP4)/2] & 0xf)];++ sumf += y[i+1] * scale.as_value * mxfp4_table[(x->qs[(i%MXFP4)/2] & 0xf0) >> 4];++ }++++++#else // defined(GGML_SIMD)++ const int nb = n / MXFP4;++ assert(n % MXFP4 == 0);++++ int yi = 0;++++ const block_mxfp4 * GGML_RESTRICT xx = (const block_mxfp4 *) vx;++++ ggml_float sumf = 0.0;++ for (int ib = 0; ib < nb; ++ib) {++ const block_mxfp4 * GGML_RESTRICT x = &xx[ib + 0];++ union {++ uint32_t as_bits;++ float as_value;++ } scale;++ scale.as_bits = (((uint32_t)x->d) << 23);++ for (int i = 0; i < MXFP4/2; ++i) {++ sumf += mxfp4_table[(x->qs[i] & 0xf)] * (ggml_float)(scale.as_value) * (ggml_float)(y[ib*MXFP4 + i*2]);++ sumf += mxfp4_table[(x->qs[i] & 0xf0) >> 4] * (ggml_float)(scale.as_value) * (ggml_float)(y[ib*MXFP4 + i*2+1]);++ }++ }++#endif++++ *s = sumf;++}+diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h+index 23cbb305..7480ca08 100644+--- a/ggml/src/ggml-cpu/vec.h++++ b/ggml/src/ggml-cpu/vec.h+@@ -42,6 +42,8 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G+ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc);+ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc);+++void ggml_vec_dot_mxfp4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);+++ void ggml_vec_silu_f32(const int n, float * y, const float * x);+ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);+ ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max);+diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu+index c6dec427..0e016ccc 100644+--- a/ggml/src/ggml-cuda/convert.cu++++ b/ggml/src/ggml-cuda/convert.cu+@@ -571,6 +571,82 @@ static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);+ }+++// MXFP4 dequantize derived from dequantize_block_q4_0++template<typename dst_t>++static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {++ const uint16_t dst_bias = 15;++ const uint16_t dst_0p5 = 0x3800;++ const uint16_t dst_m_bits = 10;++ const int64_t i = blockIdx.x;++++ // assume 32 threads++ const int64_t tid = threadIdx.x;++ const int64_t il = tid/8;++ const int64_t ir = tid%8;++ const int64_t ib = 8*i + ir;++ if (ib >= nb32) {++ return;++ }++++ const uint64_t offset = 256*i + MXFP4*ir + 8*il;++ dst_t * y = yy + offset;++++ const block_mxfp4 * x = (const block_mxfp4 *)vx + ib;++ union {++ uint32_t as_bits;++ float as_value;++ } scale;++ scale.as_bits = (((uint32_t)x->d) << 23);++++ // offset within the block 1/4 chunks (8 items)++ const uint8_t * q = x->qs + 4*il;++++ for (int l = 0; l < 4; ++l) {++ uint16_t em0 = q[l] & 0x07;++ uint16_t em1 = q[l] & 0x70;++ // float16 values++ iq1m_scale_t x0;++ iq1m_scale_t x1;++++ x0.u16 = (em0 << (dst_m_bits - 1)) | ((q[l] & 0x08) << 12);++ x1.u16 = (em1 << (dst_m_bits - 5)) | ((q[l] & 0x80) << 8);++++ // Three cases:++ // x is normal and non-zero: Correct bias++ if ((em0 & 0x06) != 0) {++ x0.u16 = x0.u16 + ((dst_bias - 1) << dst_m_bits);++ }++ if ((em1 & 0x60) != 0) {++ x1.u16 = x1.u16 + ((dst_bias - 1) << dst_m_bits);++ }++ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type++ if (em0 == 0x01) {++ x0.u16 = dst_0p5 | (x0.u16 & 0x8000);++ }++ if (em1 == 0x10) {++ x1.u16 = dst_0p5 | (x1.u16 & 0x8000);++ }++ // x is zero, do nothing++++ // XXX it looks correct here - but mulmat still gives bad results...++ // printf("i:%lld ir:%lld il:%lld l:%d y_offset:[%3lld +%d] = %f \n",++ // i, ir, il, l, 256*i + 32*ir + 4*il, l*2+ 0, scale * float(x0.f16));++ // printf("i:%lld ir:%lld il:%lld l:%d y_offset:[%3lld +%d] = %f \n",++ // i, ir, il, l, 256*i + 32*ir + 4*il, l*2+ 1, scale * float(x1.f16));++++ y[l*2] = scale.as_value * float(x0.f16);++ y[l*2+1] = scale.as_value * float(x1.f16);++ }++}++++// derived from dequantize_row_q4_0_cuda++template<typename dst_t>++static void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {++ const int nb32 = k / 32;++ const int nb = (k + 255) / 256;++ dequantize_block_mxfp4<<<nb, 32, 0, stream>>>(vx, y, nb32);++}+++ template <typename src_t, typename dst_t>+ static __global__ void convert_unary(+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01, const int64_t ne02,+@@ -664,6 +740,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ return convert_unary_cont_cuda<float>;+ case GGML_TYPE_BF16:+ return convert_unary_cont_cuda<nv_bfloat16>;++ case GGML_TYPE_MXFP4:++ return dequantize_row_mxfp4_cuda;+ default:+ return nullptr;+ }+@@ -713,6 +791,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ return convert_unary_cont_cuda<half>;+ case GGML_TYPE_BF16:+ return convert_unary_cont_cuda<nv_bfloat16>;++ case GGML_TYPE_MXFP4:++ return dequantize_row_mxfp4_cuda;+ default:+ return nullptr;+ }+diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu+index 28ccf4be..bb19b06e 100644+--- a/ggml/src/ggml-cuda/ggml-cuda.cu++++ b/ggml/src/ggml-cuda/ggml-cuda.cu+@@ -21,6 +21,7 @@+ #include "ggml-cuda/im2col.cuh"+ #include "ggml-cuda/mmq.cuh"+ #include "ggml-cuda/mmv.cuh"++#include "ggml-cuda/mmvmxfp4.cuh"+ #include "ggml-cuda/mmvq.cuh"+ #include "ggml-cuda/norm.cuh"+ #include "ggml-cuda/opt-step-adamw.cuh"+@@ -1202,7 +1203,7 @@ static void ggml_cuda_op_mul_mat_cublas(++ const int cc = ggml_cuda_info().devices[id].cc;++- const bool use_fp16 = (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT;++ const bool use_fp16 = (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT && src0->type != GGML_TYPE_MXFP4;++ if (src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) {+ ggml_cuda_pool_alloc<nv_bfloat16> src1_as_bf16(ctx.pool(id));+@@ -1924,7 +1925,11 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor+ && src0->ne[0] % 2 == 0 && src1->ne[1] == 1;+ bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32+- && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;++ && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE++ && src0->type != GGML_TYPE_MXFP4;++ bool use_mul_mat_vec_mxfp4 = src0->type == GGML_TYPE_MXFP4++ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32++ && src0->ne[0] % 2 == 0 && src1->ne[1] == 1;+ bool use_mul_mat_q = ggml_is_quantized(src0->type) && !bad_padding_clear+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;++@@ -1978,6 +1983,8 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda);+ } else if (use_mul_mat_q) {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda);++ } else if (use_mul_mat_vec_mxfp4) {++ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_mxfp4, nullptr);+ } else {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr);+ }+@@ -1997,6 +2004,10 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;++ if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {++ if (ne2 == 1 && src0->type == GGML_TYPE_MXFP4) {++ ggml_cuda_mul_mat_vec_mxfp4(ctx, src0, src1, ids, dst);++ return;++ }+ if (ne2 == 1) {+ if (ggml_is_quantized(src0->type)) {+ ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst);+@@ -3056,6 +3067,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_BF16:++ case GGML_TYPE_MXFP4:+ #ifdef GGML_USE_MUSA+ if (a->type == GGML_TYPE_Q3_K) {+ return false;+diff --git a/ggml/src/ggml-cuda/mmvmxfp4.cu b/ggml/src/ggml-cuda/mmvmxfp4.cu+new file mode 100644+index 00000000..da62062b+--- /dev/null++++ b/ggml/src/ggml-cuda/mmvmxfp4.cu+@@ -0,0 +1,307 @@++#include "ggml.h"++#include "common.cuh"++#include "mmvmxfp4.cuh"++++// MXFP4 implementation derived from mmv.cu float32 code paths++typedef union {++ half f16;++ uint16_t u16;++} f16_t;++++template <typename type_acc, int block_size> // TODO type_acc unused - consider bf16 support++static __global__ void mul_mat_vec_mxfp4(++ const block_mxfp4 * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,++ const int64_t ncols2, const int64_t nchannels_y, const int64_t stride_row,++ const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,++ const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst) {++ const int64_t row = blockIdx.x;++ const int64_t channel_dst = blockIdx.y;++ const int64_t channel_x = ids ? ids[channel_dst] : channel_dst / channel_ratio;++ const int64_t channel_y = ids ? channel_dst % nchannels_y : channel_dst;++ const int64_t sample_dst = blockIdx.z;++ const int64_t sample_x = sample_dst / sample_ratio;++ const int64_t sample_y = sample_dst;++ const int tid = threadIdx.x;++ constexpr int warp_size = ggml_cuda_get_physical_warp_size();++++ const uint16_t dst_bias = 15;++ const uint16_t dst_0p5 = 0x3800;++ const uint16_t dst_m_bits = 10;++++ x += sample_x *stride_sample_x + channel_x *stride_channel_x + row*stride_row;++ y += sample_y *stride_sample_y + channel_y *stride_channel_y;++ dst += sample_dst*stride_sample_dst + channel_dst*stride_channel_dst;++++ const float2 * y2 = (const float2 *) y;++++ extern __shared__ char data_mmv[]; // allocated in GPU shared memory: warp_size*sizeof(float)++ float * buf_iw = (float *) data_mmv;++++ if (block_size > warp_size) {++ if (tid < warp_size) {++ buf_iw[tid] = 0.0f;++ }++ __syncthreads();++ }++++ float sumf = 0.0f;++++ for (int64_t col2 = tid; col2 < ncols2; col2 += block_size) {++ int offset0 = col2 / (MXFP4/2);++ int i = col2 % (MXFP4/2);++ const block_mxfp4 *x2 = x+offset0;++++ union {++ uint32_t as_bits;++ float as_value;++ } scale;++ scale.as_bits = (((uint32_t)x2->d) << 23);++ uint16_t em0 = x2->qs[i] & 0x07;++ uint16_t em1 = x2->qs[i] & 0x70;++ // float16 values++ f16_t x0;++ f16_t x1;++ x0.u16 = (em0 << (dst_m_bits - 1)) | ((x2->qs[i] & 0x08) << 12);++ x1.u16 = (em1 << (dst_m_bits - 5)) | ((x2->qs[i] & 0x80) << 8);++++ // Three cases:++ // x is normal and non-zero: Correct bias++ if ((em0 & 0x06) != 0) {++ x0.u16 = x0.u16 + ((dst_bias - 1) << dst_m_bits);++ }++ if ((em1 & 0x60) != 0) {++ x1.u16 = x1.u16 + ((dst_bias - 1) << dst_m_bits);++ }++ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type++ if (em0 == 0x01) {++ x0.u16 = dst_0p5 | (x0.u16 & 0x8000);++ }++ if (em1 == 0x10) {++ x1.u16 = dst_0p5 | (x1.u16 & 0x8000);++ }++ // x is zero, do nothing++++ if (isnan(scale.as_value)) {++ sumf = scale.as_value;++ break;++ }++++ const float2 tmpx = {x0.f16, x1.f16};++ const float2 tmpy = y2[col2];++ sumf += tmpx.x*tmpy.x*scale.as_value;++ sumf += tmpx.y*tmpy.y*scale.as_value;++ }++++ sumf = warp_reduce_sum<warp_size>(sumf);++++ if (block_size > warp_size) {++ buf_iw[tid/warp_size] = sumf;++ __syncthreads();++ if (tid >= warp_size) {++ return;++ }++ sumf = buf_iw[tid];++ sumf = warp_reduce_sum<warp_size>(sumf);++ }++++ if (tid != 0) {++ return;++ }++++ dst[row] = sumf;++}++++template <typename type_acc>++static void launch_mul_mat_vec_cuda_mxfp4(++ const block_mxfp4 * x, const float * y, const int32_t * ids, float * dst,++ const int64_t ncols, const int64_t nrows, const int64_t stride_row, const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,++ const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,++ const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,++ cudaStream_t stream) {++ GGML_ASSERT(ncols % 2 == 0);++ // GGML_ASSERT(stride_row % 2 == 0); // TODO++ GGML_ASSERT(ids || nchannels_dst % nchannels_x == 0);++ GGML_ASSERT( nsamples_dst % nsamples_x == 0);++ const int64_t channel_ratio = nchannels_dst / nchannels_x;++ const int64_t sample_ratio = nsamples_dst / nsamples_x;++ int device;++ int warp_size;++++ CUDA_CHECK(cudaGetDevice(&device));++ warp_size = ggml_cuda_info().devices[device].warp_size;++++ int64_t block_size_best = warp_size;++ int64_t niter_best = (ncols + 2*warp_size - 1) / (2*warp_size);++ int64_t max_block_size = 256;++ if(ggml_cuda_info().devices[device].cc > GGML_CUDA_CC_OFFSET_AMD && ggml_cuda_info().devices[device].cc < GGML_CUDA_CC_RDNA1) {++ max_block_size = 128;++ }++ for (int64_t block_size = 2*warp_size; block_size <= max_block_size; block_size += warp_size) {++ const int64_t niter = (ncols + 2*block_size - 1) / (2*block_size);++ if (niter < niter_best) {++ niter_best = niter;++ block_size_best = block_size;++ }++ }++++ const int smem = warp_size*sizeof(float);++ const dim3 block_nums(nrows, nchannels_dst, nsamples_dst);++ const dim3 block_dims(block_size_best, 1, 1);++++ switch (block_size_best) {++ case 32: {++ mul_mat_vec_mxfp4<type_acc, 32><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 64: {++ mul_mat_vec_mxfp4<type_acc, 64><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 96: {++ mul_mat_vec_mxfp4<type_acc, 96><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 128: {++ mul_mat_vec_mxfp4<type_acc, 128><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 160: {++ mul_mat_vec_mxfp4<type_acc, 160><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 192: {++ mul_mat_vec_mxfp4<type_acc, 192><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 224: {++ mul_mat_vec_mxfp4<type_acc, 224><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ case 256: {++ mul_mat_vec_mxfp4<type_acc, 256><<<block_nums, block_dims, smem, stream>>>++ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,++ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);++ } break;++ default: {++ GGML_ABORT("fatal error");++ } break;++ }++}++++static void mul_mat_vec_cuda_mxfp4(++ const block_mxfp4 * x, const float * y, const int32_t * ids, float * dst,++ const int64_t ncols, const int64_t nrows, const int64_t stride_row, const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,++ const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,++ const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,++ enum ggml_prec prec, cudaStream_t stream) {++ launch_mul_mat_vec_cuda_mxfp4<float>++ (x, y, ids, dst, ncols, nrows, stride_row, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,++ stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream);++}++++void ggml_cuda_mul_mat_vec_mxfp4(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {++ GGML_ASSERT( src1->type == GGML_TYPE_F32);++ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32);++ GGML_ASSERT( dst->type == GGML_TYPE_F32);++++ GGML_TENSOR_BINARY_OP_LOCALS;++++ const size_t ts_src0 = ggml_type_size(src0->type);++ const size_t ts_src1 = ggml_type_size(src1->type);++ const size_t ts_dst = ggml_type_size(dst->type);++++ GGML_ASSERT(!ids || ne12 == 1); // Implementation is only correct for batch size 1.++ GGML_ASSERT(ne13 == ne3);++++ // GGML_ASSERT( nb00 == ts_src0); // TODO adjust for block sizing logic++ GGML_ASSERT( nb10 == ts_src1);++ GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));++ GGML_ASSERT( nb0 == ts_dst);++++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;++ const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;++++ const float * src1_d = (const float *) src1->data;++ const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;++ float * dst_d = (float *) dst->data;++++ const int64_t stride_row = src0->nb[1] / ts_src0;++ const int64_t s11 = src1->nb[1] / ts_src1;++ const int64_t s1 = dst->nb[1] / ts_dst;++ const int64_t stride_channel_x = src0->nb[2] / ts_src0;++ const int64_t s12 = src1->nb[2] / ts_src1;++ const int64_t s2 = dst->nb[2] / ts_dst;++ const int64_t stride_sample_x = src0->nb[3] / ts_src0;++ const int64_t stride_sample_y = src1->nb[3] / ts_src1;++ const int64_t stride_sample_dst = dst->nb[3] / ts_dst;++ const int64_t nsamples_dst = ne3;++ const int64_t nsamples_x = ne03;++ const int64_t nchannels_x = ne02;++ const int64_t nrows = ne01;++ const int64_t ncols = ne00;++++ // For MUL_MAT_ID the memory layout is different than for MUL_MAT:++ const int64_t ncols_dst = ids ? ne2 : ne1;++ const int64_t nchannels_y = ids ? ne11 : ne12;++ const int64_t nchannels_dst = ids ? ne1 : ne2;++ const int64_t stride_channel_dst = ids ? s1 : s2;++ const int64_t stride_channel_y = ids ? s11 : s12;++++ GGML_ASSERT(ncols_dst == 1);++++ const block_mxfp4 * src0_d = (const block_mxfp4 *) src0->data;++ mul_mat_vec_cuda_mxfp4(src0_d, src1_d, ids_d, dst_d, ncols, nrows, stride_row,++ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,++ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, prec, ctx.stream());++}++++void ggml_cuda_op_mul_mat_vec_mxfp4(++ ggml_backend_cuda_context & ctx,++ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,++ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,++ const int64_t src1_padded_row_size, cudaStream_t stream) {++++ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ GGML_ASSERT(dst->type == GGML_TYPE_F32);++++ const int64_t ne00 = src0->ne[0];++ const int64_t row_diff = row_high - row_low;++++ GGML_ASSERT(src1_ncols == 1);++++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;++ const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;++++ // ggml_cuda_op provides single, contiguous matrices++ const int64_t stride_row = ne00 / MXFP4;++ const int64_t nchannels_x = 1;++ const int64_t nchannels_y = 1;++ const int64_t nchannels_dst = 1;++ const int64_t stride_channel_x = 0;++ const int64_t stride_channel_y = 0;++ const int64_t stride_channel_dst = 0;++ const int64_t nsamples_x = 1;++ const int64_t nsamples_dst = 1;++ const int64_t stride_sample_x = 0;++ const int64_t stride_sample_y = 0;++ const int64_t stride_sample_dst = 0;++++ const block_mxfp4 * src0_d = (const block_mxfp4 *) src0_dd_i;++ mul_mat_vec_cuda_mxfp4(src0_d, src1_ddf_i, nullptr, dst_dd_i, ne00, row_diff, stride_row,++ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,++ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, prec, stream);++++ GGML_UNUSED(ctx);++ GGML_UNUSED(src1);++ GGML_UNUSED(dst);++ GGML_UNUSED(src1_ddq_i);++ GGML_UNUSED(src1_ncols);++ GGML_UNUSED(src1_padded_row_size);++}+diff --git a/ggml/src/ggml-cuda/mmvmxfp4.cuh b/ggml/src/ggml-cuda/mmvmxfp4.cuh+new file mode 100644+index 00000000..a08fc780+--- /dev/null++++ b/ggml/src/ggml-cuda/mmvmxfp4.cuh+@@ -0,0 +1,9 @@++#include "common.cuh"++++void ggml_cuda_mul_mat_vec_mxfp4(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);++++void ggml_cuda_op_mul_mat_vec_mxfp4(++ ggml_backend_cuda_context & ctx,++ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,++ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,++ const int64_t src1_padded_row_size, cudaStream_t stream);+diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h+index 17eab976..938386ba 100644+--- a/ggml/src/ggml-metal/ggml-metal-impl.h++++ b/ggml/src/ggml-metal/ggml-metal-impl.h+@@ -65,6 +65,9 @@+ #define N_R0_IQ4_XS 2+ #define N_SG_IQ4_XS 2+++#define N_R0_MXFP4 4++#define N_SG_MXFP4 2+++ // kernel argument structs+ //+ // - element counters (e.g. ne00) typically use int32_t to reduce register usage+diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m+index ab46f6e3..d8e05a21 100644+--- a/ggml/src/ggml-metal/ggml-metal.m++++ b/ggml/src/ggml-metal/ggml-metal.m+@@ -40,6 +40,7 @@ static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;+ static struct ggml_backend_reg g_ggml_backend_metal_reg;+ static struct ggml_backend_device g_ggml_backend_metal_device;++++ // information about a Metal device+ // note: assumes single GPU device - the default one+ // TODO: support multiple GPU devices+@@ -209,6 +210,7 @@ enum ggml_metal_kernel_type {+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,++ GGML_METAL_KERNEL_TYPE_MUL_MV_MXFP4_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_2,+ GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_3,+ GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_4,+@@ -288,6 +290,7 @@ enum ggml_metal_kernel_type {+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32,++ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_MXFP4_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_BF16_F32,+@@ -310,6 +313,7 @@ enum ggml_metal_kernel_type {+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32,++ GGML_METAL_KERNEL_TYPE_MUL_MM_MXFP4_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP0_F16,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F16,+@@ -334,6 +338,7 @@ enum ggml_metal_kernel_type {+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16,++ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MXFP4_F16,+ GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32,+ GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16,+ GGML_METAL_KERNEL_TYPE_ROPE_MULTI_F32,+@@ -934,7 +939,7 @@ static id<MTLLibrary> ggml_metal_load_library(id<MTLDevice> device, bool use_bfl++ MTLCompileOptions * options = [MTLCompileOptions new];+ options.preprocessorMacros = prep;+-+++ //[options setFastMathEnabled:false];++ metal_library = [device newLibraryWithSource:src options:options error:&error];+@@ -1157,6 +1162,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, has_simdgroup_reduction);++ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_MXFP4_F32, mul_mv_mxfp4_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_2, mul_mv_ext_f16_f32_r1_2, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_3, mul_mv_ext_f16_f32_r1_3, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_4, mul_mv_ext_f16_f32_r1_4, has_simdgroup_reduction);+@@ -1236,6 +1242,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, has_simdgroup_reduction);++ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_MXFP4_F32, mul_mv_id_mxfp4_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_BF16_F32, mul_mm_bf16_f32, has_simdgroup_mm && use_bfloat);+@@ -1258,6 +1265,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, has_simdgroup_mm);++ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_MXFP4_F32, mul_mm_mxfp4_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP0_F16, mul_mm_id_map0_f16, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP1_F32, mul_mm_id_map1_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F16, mul_mm_id_f32_f16, has_simdgroup_mm);+@@ -1282,6 +1290,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16, mul_mm_id_iq1_m_f16, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16, mul_mm_id_iq4_nl_f16, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16, mul_mm_id_iq4_xs_f16, has_simdgroup_mm);++ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MXFP4_F16, mul_mm_id_mxfp4_f16, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32, rope_norm_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16, rope_norm_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_MULTI_F32, rope_multi_f32, true);+@@ -3007,6 +3016,7 @@ static bool ggml_metal_encode_node(+ case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32 ].pipeline; break;+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break;+ case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32 ].pipeline; break;++ case GGML_TYPE_MXFP4: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_MXFP4_F32 ].pipeline; break;+ default: GGML_ABORT("MUL MAT-MAT not implemented");+ }++@@ -3212,6 +3222,12 @@ static bool ggml_metal_encode_node(+ smem = 32*sizeof(float);+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32].pipeline;+ } break;++ case GGML_TYPE_MXFP4:++ {++ nsg = N_SG_MXFP4;++ nr0 = N_R0_MXFP4;++ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_MXFP4_F32].pipeline;++ } break;+ default:+ {+ GGML_LOG_ERROR("Asserting on type %d\n", (int)src0t);+@@ -3396,6 +3412,7 @@ static bool ggml_metal_encode_node(+ case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16 ].pipeline; break;+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16 ].pipeline; break;+ case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16 ].pipeline; break;++ case GGML_TYPE_MXFP4: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MXFP4_F16 ].pipeline; break;+ default: GGML_ABORT("MUL_MAT_ID not implemented");+ }++@@ -3607,6 +3624,12 @@ static bool ggml_metal_encode_node(+ smem = 32*sizeof(float);+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32].pipeline;+ } break;++ case GGML_TYPE_MXFP4:++ {++ nsg = N_SG_MXFP4;++ nr0 = N_R0_MXFP4;++ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_MXFP4_F32].pipeline;++ } break;+ default:+ {+ GGML_LOG_ERROR("Asserting on type %d\n", (int)src2t);+diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal+index 08e8d807..69fa17de 100644+--- a/ggml/src/ggml-metal/ggml-metal.metal++++ b/ggml/src/ggml-metal/ggml-metal.metal+@@ -1902,16 +1902,16 @@ void mul_vec_q_n_f32_impl(+ device const char * src1,+ device char * dst,+ threadgroup char * shmem,+- uint3 tgpig,+- ushort tiisg,+- ushort sgitg) {+- const int nb = args.ne00/QK4_0;++ uint3 tgpig, // Threadgroup Position in Grid++ ushort tiisg, // Thread Index in SIMD Group++ ushort sgitg) { // SIMD Group Index in ThreadGroup++ const int nb = args.ne00/QK4_0; // src0->ne[0] / 32++ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++- const int first_row = (r0 * nsg + sgitg) * nr0;++ const int first_row = (r0 * nsg + sgitg) * nr0; // nsg=2 nr0=4++ const uint i12 = im%args.ne12;+ const uint i13 = im/args.ne12;+@@ -6744,6 +6744,49 @@ kernel void kernel_mul_mm_id(+ }+ }+++template <typename type4x4>++void dequantize_mxfp4(device const block_mxfp4 * xb, short il, thread type4x4 & reg) {++ float4x4 reg_f;++ const ushort dst_bias = 15;++ const ushort dst_0p5 = 0x3800;++ const ushort dst_m_bits = 10;++ const half scale = (half)(as_type<float>(((uint32_t)xb->d) << 23));++ // il:0 first 16, il:1 last 16++ for (int i = 0; i < 8; i++) {++ ushort em0 = xb->qs[il*8 + i] & 0x07;++ ushort em1 = xb->qs[il*8 + i] & 0x70;++ // float16 values++ ushort x0 = (em0 << (dst_m_bits - 1)) | ((xb->qs[il*8 + i] & 0x08) << 12);++ ushort x1 = (em1 << (dst_m_bits - 5)) | ((xb->qs[il*8 + i] & 0x80) << 8);++++ // Three cases:++ // x is normal and non-zero: Correct bias++ if ((em0 & 0x06) != 0) {++ x0 = x0 + ((dst_bias - 1) << dst_m_bits);++ }++ if ((em1 & 0x60) != 0) {++ x1 = x1 + ((dst_bias - 1) << dst_m_bits);++ }++ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type++ if (em0 == 0x01) {++ x0 = dst_0p5 | (x0 & 0x8000);++ }++ if (em1 == 0x10) {++ x1 = dst_0p5 | (x1 & 0x8000);++ }++ // x is zero, do nothing++++ if (isnan(scale)) {++ reg_f[i/2][2*(i%2) + 0] = scale;++ reg_f[i/2][2*(i%2) + 1] = scale;++ } else {++ reg_f[i/2][2*(i%2) + 0] = scale * as_type<half>(x0);++ reg_f[i/2][2*(i%2) + 1] = scale * as_type<half>(x1);++ }++ }++ reg = (type4x4) reg_f;++}+++ #define QK_NL 16++ //+@@ -6811,6 +6854,8 @@ template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_m+ template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;+ template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;+++template [[host_name("kernel_mul_mm_mxfp4_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4>;+++ //+ // indirect matrix-matrix multiplication+ //+@@ -6842,6 +6887,8 @@ template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_m+ template [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;+ template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;+++template [[host_name("kernel_mul_mm_id_mxfp4_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4>;++++ //+ // matrix-vector multiplication+@@ -6958,6 +7005,120 @@ kernel void kernel_mul_mv_id(+ sgitg);+ }+++// MXFP32 implementation derived from mul_vec_q_n_f32_impl and block_q_n_dot_y++void mul_mv_mxfp4_f32_impl(++ ggml_metal_kargs_mul_mv args,++ device const char * src0,++ device const char * src1,++ device char * dst,++ threadgroup char * shmem,++ uint3 tgpig,++ ushort tiisg,++ ushort sgitg) {++ const ushort dst_bias = 15;++ const ushort dst_0p5 = 0x3800;++ const ushort dst_m_bits = 10;++ const int nr0 = N_R0_MXFP4;++ const int nsg = N_SG_MXFP4;++ const int nw = N_SIMDWIDTH;++ const int nb = args.ne00/MXFP4;++++ const int r0 = tgpig.x;++ const int r1 = tgpig.y;++ const int im = tgpig.z;++++ const int first_row = (r0 * nsg + sgitg) * nr0;++++ const uint i12 = im%args.ne12;++ const uint i13 = im/args.ne12;++++ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;++++ device const float * y = (device const float *) (src1 + offset1);++++ // pointers to src0 rows++ device const block_mxfp4 * ax[nr0];++ for (int row = 0; row < nr0; ++row) {++ const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;++++ ax[row] = (device const block_mxfp4 *) ((device char *) src0 + offset0);++ }++++ float yl[16]; // src1 vector cache++ float sumf[nr0] = {0.f};++++ const short ix = (tiisg/2);++ const short il = (tiisg%2)*16;++++ device const float * yb = y + ix*MXFP4 + il;++++ // each thread in a SIMD group deals with half a block.++ for (int ib = ix; ib < nb; ib += nw/2) {++++#pragma unroll++ for (short row = 0; row < nr0; row++) {++ // Processes 16 items++ device const block_mxfp4 * qb_curr = ax[row] + ib;++ float d = as_type<float>(((uint32_t)(ax[row] + ib)->d) << 23);++ // il = 0 or 16++ device const uint8_t *qs = ((device const uint8_t *) qb_curr + 1 + il/2);++ for (int i = 0; i < 8; ++i) {++ ushort em0 = qs[i] & 0x07;++ ushort em1 = qs[i] & 0x70;++ ushort x0 = (em0 << (dst_m_bits - 1)) | ((qs[i] & 0x08) << 12);++ ushort x1 = (em1 << (dst_m_bits - 5)) | ((qs[i] & 0x80) << 8);++ // Three cases:++ // x is normal and non-zero: Correct bias++ if ((em0 & 0x06) != 0) {++ x0 = x0 + ((dst_bias - 1) << dst_m_bits);++ }++ if ((em1 & 0x60) != 0) {++ x1 = x1 + ((dst_bias - 1) << dst_m_bits);++ }++ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type++ if (em0 == 0x01) {++ x0 = dst_0p5 | (x0 & 0x8000);++ }++ if (em1 == 0x10) {++ x1 = dst_0p5 | (x1 & 0x8000);++ }++ // x is zero, do nothing++ if (!isnan(d)) {++ sumf[row] += yb[i*2] * as_type<half>(x0) * d++ + yb[i*2+1] * as_type<half>(x1) * d;++ } else {++ sumf[row] = d;++ }++ }++ }++++ yb += MXFP4 * 16;++ }++++ device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0;++++ for (int row = 0; row < nr0; ++row) {++ const float tot = simd_sum(sumf[row]);++++ if (tiisg == 0 && first_row + row < args.ne01) {++ dst_f32[first_row + row] = tot;++ }++ }++}++++[[host_name("kernel_mul_mv_mxfp4_f32")]]++kernel void kernel_mul_mv_mxfp4_f32(++ constant ggml_metal_kargs_mul_mv & args,++ device const char * src0,++ device const char * src1,++ device char * dst,++ threadgroup char * shmem [[threadgroup(0)]],++ uint3 tgpig[[threadgroup_position_in_grid]],++ ushort tiisg[[thread_index_in_simdgroup]],++ ushort sgitg[[simdgroup_index_in_threadgroup]]) {++ mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);++}+++ typedef decltype(kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>) kernel_mul_mv_id_t;++ template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>;+@@ -6987,6 +7148,8 @@ template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t+ template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL, N_SG_IQ4_NL, N_SIMDWIDTH>>>;+ template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS, N_SG_IQ4_XS, N_SIMDWIDTH>>>;+++template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_mv_mxfp4_f32_impl>>;+++ kernel void kernel_pool_2d_max_f32(+ device const float * src0,+ device float * dst,+diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c+index 84ec6dfe..17c308aa 100644+--- a/ggml/src/ggml-quants.c++++ b/ggml/src/ggml-quants.c+@@ -4925,6 +4925,144 @@ void quantize_row_iq2_s_ref(const float * GGML_RESTRICT x, block_iq2_s * GGML_RE+ quantize_iq2_s(x, y, 1, k, NULL);+ }+++// =============================== mxfp4 (de)-quantization++++void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k) {++ static const int qk = MXFP4;++ static const uint32_t E8_BIAS = 127;++ static const uint32_t E2_BIAS = 1;++++ assert(k % qk == 0);++++ const int nb = k / qk;++++ for (int i = 0; i < nb; i++) {++ float amax = 0.0f; // absolute max++++ for (int j = 0; j < qk; j++) {++ const float v = x[i*qk + j];++ if (amax < fabsf(v)) {++ amax = fabsf(v);++ }++ }++++ const float dequant_scale = amax / 6.0f;++ uint32_t dequant_scale_exponent = 0;++ memcpy(&dequant_scale_exponent, &dequant_scale, sizeof(dequant_scale_exponent));++++ // Rounding up++ dequant_scale_exponent = (dequant_scale_exponent + 0x007FFFFF) & 0x7F800000;++ // Rounding down++ // dequant_scale_exponent = dequant_scale_exponent & 0x7F800000;++++ float dequant_scale_rounded = 0.0f;++ memcpy(&dequant_scale_rounded, &dequant_scale_exponent, sizeof(dequant_scale_rounded));++ float quant_scale = 0.0f;++ if (dequant_scale_rounded != 0.0f) {++ quant_scale = 1.0f / dequant_scale_rounded;++ }++++ y[i].d = (uint8_t)(dequant_scale_exponent >> 23);++++ for (int j = 0; j < qk/2; ++j) {++ const float x0 = x[i*qk + j*2]*quant_scale;++ const float x1 = x[i*qk + j*2+1]*quant_scale;++++ uint32_t xi0 = 0;++ uint32_t xi1 = 0;++ memcpy(&xi0, &x0, sizeof(xi0));++ memcpy(&xi1, &x1, sizeof(xi1));++++ uint32_t s0 = xi0 & 0x80000000;++ uint32_t s1 = xi1 & 0x80000000;++ uint32_t e0 = (xi0 >> 23) & 0xFF;++ uint32_t e1 = (xi1 >> 23) & 0xFF;++ uint32_t m0 = (xi0 & 0x7FFFFF);++ uint32_t m1 = (xi1 & 0x7FFFFF);++++ // 0.25 <= x < 0.75 maps to 0.5, a denormal number++ // Move implicit bit 1 at the beginning to mantissa for denormals++ // adjusted_exponents++ uint32_t ae0 = E8_BIAS - (e0 + 1);++ uint32_t ae1 = E8_BIAS - (e1 + 1);++ if (e0 < E8_BIAS) {++ m0 = (0x400000 | (m0 >> 1)) >> ae0;++ }++ if (e1 < E8_BIAS) {++ m1 = (0x400000 | (m1 >> 1)) >> ae1;++ }++++ // For normal numbers, we change the bias from 127 to 1, and for subnormals, we keep exponent as 0.++ e0 = MAX(e0, E8_BIAS - E2_BIAS) - (E8_BIAS - E2_BIAS);++ e1 = MAX(e1, E8_BIAS - E2_BIAS) - (E8_BIAS - E2_BIAS);++++ // Combine sign, exponent, and mantissa, while saturating++ // rounding nearest with tie breaking up by adding +1 to one bit right of the LSB, then shift right++ uint32_t tmp0 = MIN((((e0 << 2) | (m0 >> 21)) + 1) >> 1, 0x7);++ uint32_t tmp1 = MIN((((e1 << 2) | (m1 >> 21)) + 1) >> 1, 0x7);++ uint8_t v0 = (uint8_t)((s0 >> 28) | tmp0);++ uint8_t v1 = (uint8_t)((s1 >> 28) | tmp1);++ y[i].qs[j] = v0;++ y[i].qs[j] |= v1 << 4;++ }++ }++}++++void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {++ assert(k % MXFP4 == 0);++++ const int nb = k / MXFP4;++ const uint16_t dst_bias = 15;++ const uint16_t dst_0p5 = 0x3800;++ const uint16_t dst_m_bits = 10;++++ for (int i = 0; i < nb; i++) {++ union {++ uint32_t as_bits;++ float as_value;++ } scale;++ scale.as_bits = (((uint32_t)x[i].d) << 23);++ for (int j = 0; j < MXFP4/2; ++j) {++ uint16_t em0 = x[i].qs[j] & 0x07;++ uint16_t em1 = x[i].qs[j] & 0x70;++ // float16 values++ uint16_t x0 = (em0 << (dst_m_bits - 1)) | ((x[i].qs[j] & 0x08) << 12);++ uint16_t x1 = (em1 << (dst_m_bits - 5)) | ((x[i].qs[j] & 0x80) << 8);++++ // Three cases:++ // x is normal and non-zero: Correct bias++ if ((em0 & 0x06) != 0) {++ x0 = x0 + ((dst_bias - 1) << dst_m_bits);++ }++ if ((em1 & 0x60) != 0) {++ x1 = x1 + ((dst_bias - 1) << dst_m_bits);++ }++ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type++ if (em0 == 0x01) {++ x0 = dst_0p5 | (x0 & 0x8000);++ }++ if (em1 == 0x10) {++ x1 = dst_0p5 | (x1 & 0x8000);++ }++ // x is zero, do nothing++++ if (isnan(scale.as_value)) {++ y[i*MXFP4 + j*2] = scale.as_value;++ y[i*MXFP4 + j*2+1] = scale.as_value;++ } else {++ y[i*MXFP4 + j*2] = GGML_FP16_TO_FP32(x0)*scale.as_value;++ y[i*MXFP4 + j*2+1] = GGML_FP16_TO_FP32(x1)*scale.as_value;++ }++ }++ }++}++++++size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {++ quantize_row_mxfp4_ref(src, dst, (int64_t)nrow*n_per_row);++ return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row);++}+++ // =============================== data validation++ static bool validate_float(float f, size_t i) {+@@ -5214,7 +5352,9 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);+ } break;+-++ case GGML_TYPE_MXFP4:++ // TODO - anything to validate?++ break;+ case GGML_TYPE_I8:+ case GGML_TYPE_I16:+ case GGML_TYPE_I32:+diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h+index d09173e1..2fc40f75 100644+--- a/ggml/src/ggml-quants.h++++ b/ggml/src/ggml-quants.h+@@ -37,6 +37,8 @@ GGML_API void quantize_row_iq4_xs_ref (const float * GGML_RESTRICT x, block_iq4_+ GGML_API void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);+ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);+++GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k);+++ // Dequantization+ GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+ GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+@@ -65,6 +67,8 @@ GGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, floa+ GGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+ GGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+++GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+++ // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")+ GGML_API size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+ GGML_API size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+@@ -90,6 +94,8 @@ GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTR+ GGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+ GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+++GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+++ GGML_API void iq2xs_init_impl(enum ggml_type type);+ GGML_API void iq2xs_free_impl(enum ggml_type type);+ GGML_API void iq3xs_init_impl(int grid_size);+diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c+index 8a654624..0f3c9834 100644+--- a/ggml/src/ggml.c++++ b/ggml/src/ggml.c+@@ -589,11 +589,13 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {+ .to_float = (ggml_to_float_t) dequantize_row_q4_1,+ .from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref,+ },+- [4] = { // GGML_TYPE_Q4_2+- .type_name = "DEPRECATED",+- .blck_size = 0,+- .type_size = 0,+- .is_quantized = false,++ [GGML_TYPE_MXFP4] = { // formerly deprecated GGML_TYPE_Q4_2++ .type_name = "mxfp4",++ .blck_size = MXFP4,++ .type_size = sizeof(block_mxfp4),++ .is_quantized = true,++ .to_float = (ggml_to_float_t) dequantize_row_mxfp4,++ .from_float_ref = (ggml_from_float_t) quantize_row_mxfp4_ref,+ },+ [5] = { // GGML_TYPE_Q4_3+ .type_name = "DEPRECATED",+@@ -6446,6 +6448,7 @@ size_t ggml_quantize_chunk(+ case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;++ case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_F16:+ {+ size_t elemsize = sizeof(ggml_fp16_t);diff --git a/llama/patches/0024-cuda-disable-graph-compat-check-for-OP_ADD.patch b/llama/patches/0024-cuda-disable-graph-compat-check-for-OP_ADD.patchnew file mode 100644index 000000000..535b09eb1--- /dev/null+++ b/llama/patches/0024-cuda-disable-graph-compat-check-for-OP_ADD.patch@@ -0,0 +1,34 @@+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001+From: Michael Yang <[email protected]>+Date: Thu, 31 Jul 2025 12:31:58 -0700+Subject: [PATCH] cuda: disable graph compat check for OP_ADD++---+ ggml/src/ggml-cuda/ggml-cuda.cu | 14 --------------+ 1 file changed, 14 deletions(-)++diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu+index bb19b06e..080e7467 100644+--- a/ggml/src/ggml-cuda/ggml-cuda.cu++++ b/ggml/src/ggml-cuda/ggml-cuda.cu+@@ -2509,20 +2509,6 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_backend_cud+ #endif+ }++- // workarounds to exclude Gemma3n's `project_per_layer_input` operation from the batch-size heuristic, specific to ollama's implementation of gemma3n+- // number of layers is different for per_layer_proj between gemma3n:2b and gemma3n:4b, which is why we don't check that value here+- if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1 && !(node->ne[0] == 256+- && node->ne[2] == 1+- && node->ne[3] == 1+- && node->src[0] ? std::string(node->src[0]->name).find(gemma3n_node_name) != std::string::npos : false+- && node->src[1] ? node->src[1]->name == gemma3n_per_layer_proj_src1_name : false)) {+- // Generally, changes in batch size or context size can cause changes to the grid size of some kernels.+- use_cuda_graph = false;+-#ifndef NDEBUG+- GGML_LOG_INFO("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);+-#endif+- }+-+ if (node->op == GGML_OP_CPY) {++ // Store the pointers which are updated for each token, such that these can be sentdiff --git a/llama/patches/0025-Disable-ggml-blas-on-macos-v13-and-older.patch b/llama/patches/0025-Disable-ggml-blas-on-macos-v13-and-older.patchnew file mode 100644index 000000000..465792600--- /dev/null+++ b/llama/patches/0025-Disable-ggml-blas-on-macos-v13-and-older.patch@@ -0,0 +1,25 @@+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001+From: Daniel Hiltgen <[email protected]>+Date: Sun, 3 Aug 2025 10:00:20 -0700+Subject: [PATCH] Disable ggml-blas on macos v13 and older++---+ ggml/src/ggml-blas/ggml-blas.cpp | 5 ++++++ 1 file changed, 5 insertions(+)++diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cpp+index ec158dfa..22926d75 100644+--- a/ggml/src/ggml-blas/ggml-blas.cpp++++ b/ggml/src/ggml-blas/ggml-blas.cpp+@@ -505,6 +505,11 @@ static const struct ggml_backend_reg_i ggml_backend_blas_reg_i = {+ };++ ggml_backend_reg_t ggml_backend_blas_reg(void) {++ // MacOS prior to v14 does not include cblas_sgemm - disable this backend if it isn't available++ if (&cblas_sgemm == NULL) {++ GGML_LOG_INFO("Disabling ggml-blas backend on old MacOS version\n");++ return NULL;++ }+ static struct ggml_backend_reg ggml_backend_blas_reg = {+ /* .api_version = */ GGML_BACKEND_API_VERSION,+ /* .iface = */ ggml_backend_blas_reg_i,diff --git a/ml/backend.go b/ml/backend.goindex 06f9de9ae..fcb7db5ed 100644--- a/ml/backend.go+++ b/ml/backend.go@@ -276,6 +276,7 @@ type Tensor interface {Cos(ctx Context) TensorTanh(ctx Context) TensorGELU(ctx Context) Tensor+ QuickGELU(ctx Context) TensorSILU(ctx Context) TensorRELU(ctx Context) TensorSigmoid(ctx Context) Tensor@@ -283,7 +284,7 @@ type Tensor interface {Reshape(ctx Context, shape ...int) TensorView(ctx Context, offset int, shape ...int) TensorPermute(ctx Context, shape ...int) Tensor- Contiguous(ctx Context) Tensor+ Contiguous(ctx Context, shape ...int) TensorSet(ctx Context, t2 Tensor, offset int, strides ...int) TensorPad(ctx Context, shape ...int) Tensor@@ -468,4 +469,5 @@ const (DTypeQ80DTypeQ40DTypeI32+ DTypeMXFP4)diff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.goindex 243476891..15c210dc1 100644--- a/ml/backend/ggml/ggml.go+++ b/ml/backend/ggml/ggml.go@@ -239,10 +239,12 @@ func New(modelPath string, params ml.BackendParams) (ml.Backend, error) {createTensor := func(t tensor, bts []*C.struct_ggml_backend_buffer_type, layer int) *C.struct_ggml_tensor {for _, bt := range bts {if _, ok := ctxs[bt]; !ok {+ // slog.Info("XXX before ggml_init")ctxs[bt] = C.ggml_init(C.struct_ggml_init_params{mem_size: C.ggml_tensor_overhead() * C.size_t(maxTensors),no_alloc: true,})+ // slog.Info("XXX after ggml_init")}targets[t.source.Name] = append(targets[t.source.Name], t.target)@@ -541,6 +543,8 @@ func (b *Backend) NewContextSize(n int) ml.Context {var allocatedBuffers []*C.struct_ggml_backend_buffer+ // slog.Info("XXX before ggml_init")+ // defer slog.Info("XXX after ggml_init")return &Context{b: b,maxGraphNodes: n,@@ -708,6 +712,8 @@ func (c *Context) newTensor(dtype ml.DType, shape []int) ml.Tensor {cdtype = C.GGML_TYPE_Q4_0case ml.DTypeI32:cdtype = C.GGML_TYPE_I32+ case ml.DTypeMXFP4:+ cdtype = C.GGML_TYPE_MXFP4default:panic("unsupported dtype")}@@ -896,6 +902,8 @@ func (t *Tensor) DType() ml.DType {return ml.DTypeQ40case C.GGML_TYPE_I32:return ml.DTypeI32+ case C.GGML_TYPE_MXFP4:+ return ml.DTypeMXFP4default:return ml.DTypeOther}@@ -958,10 +966,35 @@ func (t *Tensor) Concat(ctx ml.Context, t2 ml.Tensor, dim int) ml.Tensor {}}-func (t *Tensor) Contiguous(ctx ml.Context) ml.Tensor {- return &Tensor{- b: t.b,- t: C.ggml_cont(ctx.(*Context).ctx, t.t),+func (t *Tensor) Contiguous(ctx ml.Context, shape ...int) ml.Tensor {+ switch len(shape) {+ case 0:+ return &Tensor{+ b: t.b,+ t: C.ggml_cont(ctx.(*Context).ctx, t.t),+ }+ case 1:+ return &Tensor{+ b: t.b,+ t: C.ggml_cont_1d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0])),+ }+ case 2:+ return &Tensor{+ b: t.b,+ t: C.ggml_cont_2d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1])),+ }+ case 3:+ return &Tensor{+ b: t.b,+ t: C.ggml_cont_3d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1]), C.int64_t(shape[2])),+ }+ case 4:+ return &Tensor{+ b: t.b,+ t: C.ggml_cont_4d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1]), C.int64_t(shape[2]), C.int64_t(shape[3])),+ }+ default:+ panic("unsupported number of dimensions")}}@@ -1176,11 +1209,18 @@ func (t *Tensor) View(ctx ml.Context, offset int, shape ...int) ml.Tensor {func (t *Tensor) RoPE(ctx ml.Context, positions ml.Tensor, ropeDim int, ropeBase, ropeScale float32, options ...func(*rope.Options)) ml.Tensor {// Default options- opts := &rope.Options{OriginalContextLength: 131072, Factors: &Tensor{}}+ opts := rope.Options{+ Factors: &Tensor{},+ OriginalContextLength: 131072,+ ExtrapolationFactor: 0.,+ AttentionFactor: 1.,+ BetaFast: 32.,+ BetaSlow: 1.,+ }// Apply any provided optionsfor _, option := range options {- option(opts)+ option(&opts)}dequant := t.t@@ -1200,10 +1240,10 @@ func (t *Tensor) RoPE(ctx ml.Context, positions ml.Tensor, ropeDim int, ropeBaseC.int(opts.OriginalContextLength),C.float(ropeBase),C.float(ropeScale),- C.float(0.0),- C.float(1.0),- C.float(32.0),- C.float(1.0),+ C.float(opts.ExtrapolationFactor),+ C.float(opts.AttentionFactor),+ C.float(opts.BetaFast),+ C.float(opts.BetaSlow),),}}@@ -1222,6 +1262,13 @@ func (t *Tensor) GELU(ctx ml.Context) ml.Tensor {}}+func (t *Tensor) QuickGELU(ctx ml.Context) ml.Tensor {+ return &Tensor{+ b: t.b,+ t: C.ggml_gelu_quick_inplace(ctx.(*Context).ctx, t.t),+ }+}+func (t *Tensor) SILU(ctx ml.Context) ml.Tensor {return &Tensor{b: t.b,@@ -1350,3 +1397,65 @@ func (t *Tensor) Clamp(ctx ml.Context, min, max float32) ml.Tensor {t: C.ggml_clamp(ctx.(*Context).ctx, t.t, C.float(min), C.float(max)),}}++func (c Context) FromBytes(dtype ml.DType, s []uint8, shape ...int) ml.Tensor {+ // Unchecked to handle quantized types+ t := c.newTensor(dtype, shape)+ if len(s) > 0 {+ C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t))+ }++ return t+}++// TODO - DRY this out with New if possible+func newTestBackend(size int) *Backend {+ var cpus []*C.struct_ggml_backend_device+ for _, d := range devices() {+ switch C.ggml_backend_dev_type(d) {+ case C.GGML_BACKEND_DEVICE_TYPE_CPU:+ if len(cpus) == 0 {+ // only the first cpu device should be used+ cpus = append(cpus, d)+ break+ }+ }+ }+ var schedBackends []*C.struct_ggml_backend+ var schedBufts []*C.struct_ggml_backend_buffer_type+ b := C.ggml_backend_dev_init(cpus[0], nil)+ bt := C.ggml_backend_get_default_buffer_type(b)+ C.ggml_backend_cpu_set_n_threads(b, C.int(Threads(runtime.NumCPU())))+ // C.ggml_backend_cpu_set_n_threads(b, 1) // DEBUGGING+ schedBackends = append(schedBackends, b)+ schedBufts = append(schedBufts, bt)+ return &Backend{+ meta: nil,+ sched: C.ggml_backend_sched_new(+ (*C.ggml_backend_t)(unsafe.Pointer(&schedBackends[0])),+ (*C.ggml_backend_buffer_type_t)(unsafe.Pointer(&schedBufts[0])),+ C.int(len(schedBackends)),+ C.size_t(max(8192, size)),+ false,+ false,+ ),+ input: bt,+ maxGraphNodes: max(8192, size),+ schedBackends: schedBackends,+ schedBufts: schedBufts,+ }+}++func newTestContext(b *Backend, n int) *Context {+ n = max(8192, n)+ // slog.Info("XXX before ggml_init")+ // defer slog.Info("XXX after ggml_init")+ return &Context{+ b: b,+ maxGraphNodes: n,+ ctx: C.ggml_init(C.struct_ggml_init_params{+ mem_size: C.size_t(n)*C.ggml_tensor_overhead() + C.ggml_graph_overhead_custom(C.size_t(n), false),+ no_alloc: true,+ }),+ }+}diff --git a/ml/backend/ggml/ggml/include/ggml.h b/ml/backend/ggml/ggml/include/ggml.hindex e91dedf14..873baa24f 100644--- a/ml/backend/ggml/ggml/include/ggml.h+++ b/ml/backend/ggml/ggml/include/ggml.h@@ -353,7 +353,7 @@ extern "C" {GGML_TYPE_F16 = 1,GGML_TYPE_Q4_0 = 2,GGML_TYPE_Q4_1 = 3,- // GGML_TYPE_Q4_2 = 4, support has been removed+ GGML_TYPE_MXFP4 = 4, // Formerly removed type GGML_TYPE_Q4_2// GGML_TYPE_Q4_3 = 5, support has been removedGGML_TYPE_Q5_0 = 6,GGML_TYPE_Q5_1 = 7,diff --git a/ml/backend/ggml/ggml/src/ggml-blas/ggml-blas.cpp b/ml/backend/ggml/ggml/src/ggml-blas/ggml-blas.cppindex ec158dfac..22926d758 100644--- a/ml/backend/ggml/ggml/src/ggml-blas/ggml-blas.cpp+++ b/ml/backend/ggml/ggml/src/ggml-blas/ggml-blas.cpp@@ -505,6 +505,11 @@ static const struct ggml_backend_reg_i ggml_backend_blas_reg_i = {};ggml_backend_reg_t ggml_backend_blas_reg(void) {+ // MacOS prior to v14 does not include cblas_sgemm - disable this backend if it isn't available+ if (&cblas_sgemm == NULL) {+ GGML_LOG_INFO("Disabling ggml-blas backend on old MacOS version\n");+ return NULL;+ }static struct ggml_backend_reg ggml_backend_blas_reg = {/* .api_version = */ GGML_BACKEND_API_VERSION,/* .iface = */ ggml_backend_blas_reg_i,diff --git a/ml/backend/ggml/ggml/src/ggml-common.h b/ml/backend/ggml/ggml/src/ggml-common.hindex 086c822d7..e0d71451b 100644--- a/ml/backend/ggml/ggml/src/ggml-common.h+++ b/ml/backend/ggml/ggml/src/ggml-common.h@@ -417,6 +417,13 @@ typedef struct {} block_iq4_xs;static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");+#define MXFP4 32+typedef struct {+ uint8_t d; // scale E8M0 float+ uint8_t qs[MXFP4 / 2]; // (32) 4 bit elements E2M1 float+} block_mxfp4;+static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + MXFP4/2, "wrong mxfp4 block size/padding");+#endif // GGML_COMMON_DECL#endif // GGML_COMMON_DECLdiff --git a/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu-quants.h b/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu-quants.hindex e33d9d473..6a25d0626 100644--- a/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu-quants.h+++ b/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu-quants.h@@ -58,6 +58,8 @@ void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, constvoid ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_mxfp4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);+#ifdef __cplusplus}#endifdiff --git a/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu.c b/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu.cindex 2462d2b85..bff9c426e 100644--- a/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu.c@@ -362,6 +362,11 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {.vec_dot_type = GGML_TYPE_Q8_K,.nrows = 1,},+ [GGML_TYPE_MXFP4] = {+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_mxfp4,+ .vec_dot_type = GGML_TYPE_F32,+ .nrows = 1,+ },};const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {diff --git a/ml/backend/ggml/ggml/src/ggml-cpu/ops.cpp b/ml/backend/ggml/ggml/src/ggml-cpu/ops.cppindex 654e2f280..be0aa683b 100644--- a/ml/backend/ggml/ggml/src/ggml-cpu/ops.cpp+++ b/ml/backend/ggml/ggml/src/ggml-cpu/ops.cpp@@ -4965,6 +4965,7 @@ void ggml_compute_forward_clamp(case GGML_TYPE_I32:case GGML_TYPE_I64:case GGML_TYPE_F64:+ case GGML_TYPE_MXFP4:case GGML_TYPE_COUNT:{GGML_ABORT("fatal error");diff --git a/ml/backend/ggml/ggml/src/ggml-cpu/vec.cpp b/ml/backend/ggml/ggml/src/ggml-cpu/vec.cppindex 02d406182..ec3ec9b17 100644--- a/ml/backend/ggml/ggml/src/ggml-cpu/vec.cpp+++ b/ml/backend/ggml/ggml/src/ggml-cpu/vec.cpp@@ -250,3 +250,93 @@ ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, fl}return sum = (ggml_float)logf(sum);}++#define MXFP4 32+typedef struct {+ uint8_t d; // scale E8M0 float+ uint8_t qs[MXFP4 / 2]; // (32) 4 bit elements E2M1 float+} block_mxfp4;+static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + MXFP4/2, "wrong mxfp4 block size/padding");+#define MXFP4_VALS {0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, 0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0}++void ggml_vec_dot_mxfp4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {+ assert(nrc == 1);+ GGML_UNUSED(nrc);+ GGML_UNUSED(bx);+ GGML_UNUSED(by);+ GGML_UNUSED(bs);+ ggml_float mxfp4_table[] = MXFP4_VALS;++#if defined(GGML_SIMD)+ float sumf = 0.0f;+ const int np = (n & ~(GGML_F32_STEP - 1));+ const block_mxfp4 * GGML_RESTRICT xx = (const block_mxfp4 *) vx;+ GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };++ GGML_F32_VEC scalev;+ GGML_F32_VEC ax[GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];+ for (int i = 0; i < np; i += GGML_F32_STEP) { // ARM: +16 AVX512: +64+ for (int j = 0; j < GGML_F32_ARR; j++) { // ARM: 0 .. 4 AVX512: 0 .. 4+ // convert GGML_F32_ARR X elements+ const int ib = (i + j*GGML_F32_EPR) / MXFP4;+ const block_mxfp4 * GGML_RESTRICT x = &xx[ib];+ union {+ uint32_t as_bits;+ float as_value;+ } scale;+ scale.as_bits = (((uint32_t)x->d) << 23);+ scalev = GGML_F32_VEC_SET1(scale.as_value);+ float xf[GGML_F32_EPR]= {0.f};+ assert(((i+j*GGML_F32_EPR) % MXFP4)+GGML_F32_ARR < MXFP4 && "block overrun");+ for (int qi = 0; qi < GGML_F32_EPR/2 ; ++qi) {+ xf[qi*2] = mxfp4_table[(x->qs[((i+j*GGML_F32_EPR)%MXFP4)/2+qi] & 0xf)];+ xf[qi*2+1] = mxfp4_table[(x->qs[((i+j*GGML_F32_EPR)%MXFP4)/2+qi] & 0xf0) >> 4];+ }++ ax[j] = GGML_F32_VEC_MUL(GGML_F32_VEC_LOAD(xf), scalev);+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);+ sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);+ }+ }+ GGML_F32_VEC_REDUCE(sumf, sum);++ // leftovers+ for (int i = np; i < n; i+=2) {+ const int ib = i / MXFP4;+ const block_mxfp4 * GGML_RESTRICT x = &xx[ib];+ union {+ uint32_t as_bits;+ float as_value;+ } scale;+ scale.as_bits = (((uint32_t)x->d) << 23);+ sumf += y[i] * scale.as_value * mxfp4_table[(x->qs[(i%MXFP4)/2] & 0xf)];+ sumf += y[i+1] * scale.as_value * mxfp4_table[(x->qs[(i%MXFP4)/2] & 0xf0) >> 4];+ }+++#else // defined(GGML_SIMD)+ const int nb = n / MXFP4;+ assert(n % MXFP4 == 0);++ int yi = 0;++ const block_mxfp4 * GGML_RESTRICT xx = (const block_mxfp4 *) vx;++ ggml_float sumf = 0.0;+ for (int ib = 0; ib < nb; ++ib) {+ const block_mxfp4 * GGML_RESTRICT x = &xx[ib + 0];+ union {+ uint32_t as_bits;+ float as_value;+ } scale;+ scale.as_bits = (((uint32_t)x->d) << 23);+ for (int i = 0; i < MXFP4/2; ++i) {+ sumf += mxfp4_table[(x->qs[i] & 0xf)] * (ggml_float)(scale.as_value) * (ggml_float)(y[ib*MXFP4 + i*2]);+ sumf += mxfp4_table[(x->qs[i] & 0xf0) >> 4] * (ggml_float)(scale.as_value) * (ggml_float)(y[ib*MXFP4 + i*2+1]);+ }+ }+#endif++ *s = sumf;+}diff --git a/ml/backend/ggml/ggml/src/ggml-cpu/vec.h b/ml/backend/ggml/ggml/src/ggml-cpu/vec.hindex 23cbb3051..7480ca089 100644--- a/ml/backend/ggml/ggml/src/ggml-cpu/vec.h+++ b/ml/backend/ggml/ggml/src/ggml-cpu/vec.h@@ -42,6 +42,8 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * Gvoid ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc);void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc);+void ggml_vec_dot_mxfp4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);+void ggml_vec_silu_f32(const int n, float * y, const float * x);ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);ggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max);diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/convert.cu b/ml/backend/ggml/ggml/src/ggml-cuda/convert.cuindex c6dec4276..0e016ccc0 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/convert.cu+++ b/ml/backend/ggml/ggml/src/ggml-cuda/convert.cu@@ -571,6 +571,82 @@ static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_tdequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);}+// MXFP4 dequantize derived from dequantize_block_q4_0+template<typename dst_t>+static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {+ const uint16_t dst_bias = 15;+ const uint16_t dst_0p5 = 0x3800;+ const uint16_t dst_m_bits = 10;+ const int64_t i = blockIdx.x;++ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t ib = 8*i + ir;+ if (ib >= nb32) {+ return;+ }++ const uint64_t offset = 256*i + MXFP4*ir + 8*il;+ dst_t * y = yy + offset;++ const block_mxfp4 * x = (const block_mxfp4 *)vx + ib;+ union {+ uint32_t as_bits;+ float as_value;+ } scale;+ scale.as_bits = (((uint32_t)x->d) << 23);++ // offset within the block 1/4 chunks (8 items)+ const uint8_t * q = x->qs + 4*il;++ for (int l = 0; l < 4; ++l) {+ uint16_t em0 = q[l] & 0x07;+ uint16_t em1 = q[l] & 0x70;+ // float16 values+ iq1m_scale_t x0;+ iq1m_scale_t x1;++ x0.u16 = (em0 << (dst_m_bits - 1)) | ((q[l] & 0x08) << 12);+ x1.u16 = (em1 << (dst_m_bits - 5)) | ((q[l] & 0x80) << 8);++ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0.u16 = x0.u16 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1.u16 = x1.u16 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0.u16 = dst_0p5 | (x0.u16 & 0x8000);+ }+ if (em1 == 0x10) {+ x1.u16 = dst_0p5 | (x1.u16 & 0x8000);+ }+ // x is zero, do nothing++ // XXX it looks correct here - but mulmat still gives bad results...+ // printf("i:%lld ir:%lld il:%lld l:%d y_offset:[%3lld +%d] = %f \n",+ // i, ir, il, l, 256*i + 32*ir + 4*il, l*2+ 0, scale * float(x0.f16));+ // printf("i:%lld ir:%lld il:%lld l:%d y_offset:[%3lld +%d] = %f \n",+ // i, ir, il, l, 256*i + 32*ir + 4*il, l*2+ 1, scale * float(x1.f16));++ y[l*2] = scale.as_value * float(x0.f16);+ y[l*2+1] = scale.as_value * float(x1.f16);+ }+}++// derived from dequantize_row_q4_0_cuda+template<typename dst_t>+static void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_mxfp4<<<nb, 32, 0, stream>>>(vx, y, nb32);+}+template <typename src_t, typename dst_t>static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01, const int64_t ne02,@@ -664,6 +740,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return convert_unary_cont_cuda<float>;case GGML_TYPE_BF16:return convert_unary_cont_cuda<nv_bfloat16>;+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;default:return nullptr;}@@ -713,6 +791,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {return convert_unary_cont_cuda<half>;case GGML_TYPE_BF16:return convert_unary_cont_cuda<nv_bfloat16>;+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;default:return nullptr;}diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cuindex 28ccf4bef..080e7467b 100644--- a/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu@@ -21,6 +21,7 @@#include "ggml-cuda/im2col.cuh"#include "ggml-cuda/mmq.cuh"#include "ggml-cuda/mmv.cuh"+#include "ggml-cuda/mmvmxfp4.cuh"#include "ggml-cuda/mmvq.cuh"#include "ggml-cuda/norm.cuh"#include "ggml-cuda/opt-step-adamw.cuh"@@ -1202,7 +1203,7 @@ static void ggml_cuda_op_mul_mat_cublas(const int cc = ggml_cuda_info().devices[id].cc;- const bool use_fp16 = (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT;+ const bool use_fp16 = (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT && src0->type != GGML_TYPE_MXFP4;if (src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) {ggml_cuda_pool_alloc<nv_bfloat16> src1_as_bf16(ctx.pool(id));@@ -1924,7 +1925,11 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor&& src0->ne[0] % 2 == 0 && src1->ne[1] == 1;bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32- && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;+ && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE+ && src0->type != GGML_TYPE_MXFP4;+ bool use_mul_mat_vec_mxfp4 = src0->type == GGML_TYPE_MXFP4+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32+ && src0->ne[0] % 2 == 0 && src1->ne[1] == 1;bool use_mul_mat_q = ggml_is_quantized(src0->type) && !bad_padding_clear&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;@@ -1978,6 +1983,8 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensorggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda);} else if (use_mul_mat_q) {ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda);+ } else if (use_mul_mat_vec_mxfp4) {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_mxfp4, nullptr);} else {ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr);}@@ -1997,6 +2004,10 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {+ if (ne2 == 1 && src0->type == GGML_TYPE_MXFP4) {+ ggml_cuda_mul_mat_vec_mxfp4(ctx, src0, src1, ids, dst);+ return;+ }if (ne2 == 1) {if (ggml_is_quantized(src0->type)) {ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst);@@ -2498,20 +2509,6 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_backend_cud#endif}- // workarounds to exclude Gemma3n's `project_per_layer_input` operation from the batch-size heuristic, specific to ollama's implementation of gemma3n- // number of layers is different for per_layer_proj between gemma3n:2b and gemma3n:4b, which is why we don't check that value here- if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1 && !(node->ne[0] == 256- && node->ne[2] == 1- && node->ne[3] == 1- && node->src[0] ? std::string(node->src[0]->name).find(gemma3n_node_name) != std::string::npos : false- && node->src[1] ? node->src[1]->name == gemma3n_per_layer_proj_src1_name : false)) {- // Generally, changes in batch size or context size can cause changes to the grid size of some kernels.- use_cuda_graph = false;-#ifndef NDEBUG- GGML_LOG_INFO("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);-#endif- }-if (node->op == GGML_OP_CPY) {// Store the pointers which are updated for each token, such that these can be sent@@ -3056,6 +3053,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_TYPE_IQ4_NL:case GGML_TYPE_IQ4_XS:case GGML_TYPE_BF16:+ case GGML_TYPE_MXFP4:#ifdef GGML_USE_MUSAif (a->type == GGML_TYPE_Q3_K) {return false;diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/mmvmxfp4.cu b/ml/backend/ggml/ggml/src/ggml-cuda/mmvmxfp4.cunew file mode 100644index 000000000..da62062b3--- /dev/null+++ b/ml/backend/ggml/ggml/src/ggml-cuda/mmvmxfp4.cu@@ -0,0 +1,307 @@+#include "ggml.h"+#include "common.cuh"+#include "mmvmxfp4.cuh"++// MXFP4 implementation derived from mmv.cu float32 code paths+typedef union {+ half f16;+ uint16_t u16;+} f16_t;++template <typename type_acc, int block_size> // TODO type_acc unused - consider bf16 support+static __global__ void mul_mat_vec_mxfp4(+ const block_mxfp4 * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,+ const int64_t ncols2, const int64_t nchannels_y, const int64_t stride_row,+ const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,+ const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst) {+ const int64_t row = blockIdx.x;+ const int64_t channel_dst = blockIdx.y;+ const int64_t channel_x = ids ? ids[channel_dst] : channel_dst / channel_ratio;+ const int64_t channel_y = ids ? channel_dst % nchannels_y : channel_dst;+ const int64_t sample_dst = blockIdx.z;+ const int64_t sample_x = sample_dst / sample_ratio;+ const int64_t sample_y = sample_dst;+ const int tid = threadIdx.x;+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();++ const uint16_t dst_bias = 15;+ const uint16_t dst_0p5 = 0x3800;+ const uint16_t dst_m_bits = 10;++ x += sample_x *stride_sample_x + channel_x *stride_channel_x + row*stride_row;+ y += sample_y *stride_sample_y + channel_y *stride_channel_y;+ dst += sample_dst*stride_sample_dst + channel_dst*stride_channel_dst;++ const float2 * y2 = (const float2 *) y;++ extern __shared__ char data_mmv[]; // allocated in GPU shared memory: warp_size*sizeof(float)+ float * buf_iw = (float *) data_mmv;++ if (block_size > warp_size) {+ if (tid < warp_size) {+ buf_iw[tid] = 0.0f;+ }+ __syncthreads();+ }++ float sumf = 0.0f;++ for (int64_t col2 = tid; col2 < ncols2; col2 += block_size) {+ int offset0 = col2 / (MXFP4/2);+ int i = col2 % (MXFP4/2);+ const block_mxfp4 *x2 = x+offset0;++ union {+ uint32_t as_bits;+ float as_value;+ } scale;+ scale.as_bits = (((uint32_t)x2->d) << 23);+ uint16_t em0 = x2->qs[i] & 0x07;+ uint16_t em1 = x2->qs[i] & 0x70;+ // float16 values+ f16_t x0;+ f16_t x1;+ x0.u16 = (em0 << (dst_m_bits - 1)) | ((x2->qs[i] & 0x08) << 12);+ x1.u16 = (em1 << (dst_m_bits - 5)) | ((x2->qs[i] & 0x80) << 8);++ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0.u16 = x0.u16 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1.u16 = x1.u16 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0.u16 = dst_0p5 | (x0.u16 & 0x8000);+ }+ if (em1 == 0x10) {+ x1.u16 = dst_0p5 | (x1.u16 & 0x8000);+ }+ // x is zero, do nothing++ if (isnan(scale.as_value)) {+ sumf = scale.as_value;+ break;+ }++ const float2 tmpx = {x0.f16, x1.f16};+ const float2 tmpy = y2[col2];+ sumf += tmpx.x*tmpy.x*scale.as_value;+ sumf += tmpx.y*tmpy.y*scale.as_value;+ }++ sumf = warp_reduce_sum<warp_size>(sumf);++ if (block_size > warp_size) {+ buf_iw[tid/warp_size] = sumf;+ __syncthreads();+ if (tid >= warp_size) {+ return;+ }+ sumf = buf_iw[tid];+ sumf = warp_reduce_sum<warp_size>(sumf);+ }++ if (tid != 0) {+ return;+ }++ dst[row] = sumf;+}++template <typename type_acc>+static void launch_mul_mat_vec_cuda_mxfp4(+ const block_mxfp4 * x, const float * y, const int32_t * ids, float * dst,+ const int64_t ncols, const int64_t nrows, const int64_t stride_row, const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,+ const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,+ const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,+ cudaStream_t stream) {+ GGML_ASSERT(ncols % 2 == 0);+ // GGML_ASSERT(stride_row % 2 == 0); // TODO+ GGML_ASSERT(ids || nchannels_dst % nchannels_x == 0);+ GGML_ASSERT( nsamples_dst % nsamples_x == 0);+ const int64_t channel_ratio = nchannels_dst / nchannels_x;+ const int64_t sample_ratio = nsamples_dst / nsamples_x;+ int device;+ int warp_size;++ CUDA_CHECK(cudaGetDevice(&device));+ warp_size = ggml_cuda_info().devices[device].warp_size;++ int64_t block_size_best = warp_size;+ int64_t niter_best = (ncols + 2*warp_size - 1) / (2*warp_size);+ int64_t max_block_size = 256;+ if(ggml_cuda_info().devices[device].cc > GGML_CUDA_CC_OFFSET_AMD && ggml_cuda_info().devices[device].cc < GGML_CUDA_CC_RDNA1) {+ max_block_size = 128;+ }+ for (int64_t block_size = 2*warp_size; block_size <= max_block_size; block_size += warp_size) {+ const int64_t niter = (ncols + 2*block_size - 1) / (2*block_size);+ if (niter < niter_best) {+ niter_best = niter;+ block_size_best = block_size;+ }+ }++ const int smem = warp_size*sizeof(float);+ const dim3 block_nums(nrows, nchannels_dst, nsamples_dst);+ const dim3 block_dims(block_size_best, 1, 1);++ switch (block_size_best) {+ case 32: {+ mul_mat_vec_mxfp4<type_acc, 32><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 64: {+ mul_mat_vec_mxfp4<type_acc, 64><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 96: {+ mul_mat_vec_mxfp4<type_acc, 96><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 128: {+ mul_mat_vec_mxfp4<type_acc, 128><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 160: {+ mul_mat_vec_mxfp4<type_acc, 160><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 192: {+ mul_mat_vec_mxfp4<type_acc, 192><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 224: {+ mul_mat_vec_mxfp4<type_acc, 224><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ case 256: {+ mul_mat_vec_mxfp4<type_acc, 256><<<block_nums, block_dims, smem, stream>>>+ (x, y, ids, dst, ncols/2, nchannels_y, stride_row, channel_ratio, stride_channel_x, stride_channel_y,+ stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);+ } break;+ default: {+ GGML_ABORT("fatal error");+ } break;+ }+}++static void mul_mat_vec_cuda_mxfp4(+ const block_mxfp4 * x, const float * y, const int32_t * ids, float * dst,+ const int64_t ncols, const int64_t nrows, const int64_t stride_row, const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,+ const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,+ const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,+ enum ggml_prec prec, cudaStream_t stream) {+ launch_mul_mat_vec_cuda_mxfp4<float>+ (x, y, ids, dst, ncols, nrows, stride_row, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,+ stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream);+}++void ggml_cuda_mul_mat_vec_mxfp4(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {+ GGML_ASSERT( src1->type == GGML_TYPE_F32);+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ GGML_TENSOR_BINARY_OP_LOCALS;++ const size_t ts_src0 = ggml_type_size(src0->type);+ const size_t ts_src1 = ggml_type_size(src1->type);+ const size_t ts_dst = ggml_type_size(dst->type);++ GGML_ASSERT(!ids || ne12 == 1); // Implementation is only correct for batch size 1.+ GGML_ASSERT(ne13 == ne3);++ // GGML_ASSERT( nb00 == ts_src0); // TODO adjust for block sizing logic+ GGML_ASSERT( nb10 == ts_src1);+ GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));+ GGML_ASSERT( nb0 == ts_dst);++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;+ const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;++ const float * src1_d = (const float *) src1->data;+ const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;+ float * dst_d = (float *) dst->data;++ const int64_t stride_row = src0->nb[1] / ts_src0;+ const int64_t s11 = src1->nb[1] / ts_src1;+ const int64_t s1 = dst->nb[1] / ts_dst;+ const int64_t stride_channel_x = src0->nb[2] / ts_src0;+ const int64_t s12 = src1->nb[2] / ts_src1;+ const int64_t s2 = dst->nb[2] / ts_dst;+ const int64_t stride_sample_x = src0->nb[3] / ts_src0;+ const int64_t stride_sample_y = src1->nb[3] / ts_src1;+ const int64_t stride_sample_dst = dst->nb[3] / ts_dst;+ const int64_t nsamples_dst = ne3;+ const int64_t nsamples_x = ne03;+ const int64_t nchannels_x = ne02;+ const int64_t nrows = ne01;+ const int64_t ncols = ne00;++ // For MUL_MAT_ID the memory layout is different than for MUL_MAT:+ const int64_t ncols_dst = ids ? ne2 : ne1;+ const int64_t nchannels_y = ids ? ne11 : ne12;+ const int64_t nchannels_dst = ids ? ne1 : ne2;+ const int64_t stride_channel_dst = ids ? s1 : s2;+ const int64_t stride_channel_y = ids ? s11 : s12;++ GGML_ASSERT(ncols_dst == 1);++ const block_mxfp4 * src0_d = (const block_mxfp4 *) src0->data;+ mul_mat_vec_cuda_mxfp4(src0_d, src1_d, ids_d, dst_d, ncols, nrows, stride_row,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, prec, ctx.stream());+}++void ggml_cuda_op_mul_mat_vec_mxfp4(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {++ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);++ const int64_t ne00 = src0->ne[0];+ const int64_t row_diff = row_high - row_low;++ GGML_ASSERT(src1_ncols == 1);++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;+ const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;++ // ggml_cuda_op provides single, contiguous matrices+ const int64_t stride_row = ne00 / MXFP4;+ const int64_t nchannels_x = 1;+ const int64_t nchannels_y = 1;+ const int64_t nchannels_dst = 1;+ const int64_t stride_channel_x = 0;+ const int64_t stride_channel_y = 0;+ const int64_t stride_channel_dst = 0;+ const int64_t nsamples_x = 1;+ const int64_t nsamples_dst = 1;+ const int64_t stride_sample_x = 0;+ const int64_t stride_sample_y = 0;+ const int64_t stride_sample_dst = 0;++ const block_mxfp4 * src0_d = (const block_mxfp4 *) src0_dd_i;+ mul_mat_vec_cuda_mxfp4(src0_d, src1_ddf_i, nullptr, dst_dd_i, ne00, row_diff, stride_row,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, prec, stream);++ GGML_UNUSED(ctx);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_ddq_i);+ GGML_UNUSED(src1_ncols);+ GGML_UNUSED(src1_padded_row_size);+}diff --git a/ml/backend/ggml/ggml/src/ggml-cuda/mmvmxfp4.cuh b/ml/backend/ggml/ggml/src/ggml-cuda/mmvmxfp4.cuhnew file mode 100644index 000000000..a08fc7800--- /dev/null+++ b/ml/backend/ggml/ggml/src/ggml-cuda/mmvmxfp4.cuh@@ -0,0 +1,9 @@+#include "common.cuh"++void ggml_cuda_mul_mat_vec_mxfp4(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);++void ggml_cuda_op_mul_mat_vec_mxfp4(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream);diff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.metal b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.metalindex 8f9a25e6f..5eba1dafc 100644--- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.metal+++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.metal@@ -421,6 +421,13 @@ typedef struct {} block_iq4_xs;static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");+#define MXFP4 32+typedef struct {+ uint8_t d; // scale E8M0 float+ uint8_t qs[MXFP4 / 2]; // (32) 4 bit elements E2M1 float+} block_mxfp4;+static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + MXFP4/2, "wrong mxfp4 block size/padding");+#endif // GGML_COMMON_DECL#endif // GGML_COMMON_DECL@@ -1929,6 +1936,9 @@ GGML_TABLE_END()#define N_R0_IQ4_XS 2#define N_SG_IQ4_XS 2+#define N_R0_MXFP4 4+#define N_SG_MXFP4 2+// kernel argument structs//// - element counters (e.g. ne00) typically use int32_t to reduce register usage@@ -4380,16 +4390,16 @@ void mul_vec_q_n_f32_impl(device const char * src1,device char * dst,threadgroup char * shmem,- uint3 tgpig,- ushort tiisg,- ushort sgitg) {- const int nb = args.ne00/QK4_0;+ uint3 tgpig, // Threadgroup Position in Grid+ ushort tiisg, // Thread Index in SIMD Group+ ushort sgitg) { // SIMD Group Index in ThreadGroup+ const int nb = args.ne00/QK4_0; // src0->ne[0] / 32const int r0 = tgpig.x;const int r1 = tgpig.y;const int im = tgpig.z;- const int first_row = (r0 * nsg + sgitg) * nr0;+ const int first_row = (r0 * nsg + sgitg) * nr0; // nsg=2 nr0=4const uint i12 = im%args.ne12;const uint i13 = im/args.ne12;@@ -9222,6 +9232,49 @@ kernel void kernel_mul_mm_id(}}+template <typename type4x4>+void dequantize_mxfp4(device const block_mxfp4 * xb, short il, thread type4x4 & reg) {+ float4x4 reg_f;+ const ushort dst_bias = 15;+ const ushort dst_0p5 = 0x3800;+ const ushort dst_m_bits = 10;+ const half scale = (half)(as_type<float>(((uint32_t)xb->d) << 23));+ // il:0 first 16, il:1 last 16+ for (int i = 0; i < 8; i++) {+ ushort em0 = xb->qs[il*8 + i] & 0x07;+ ushort em1 = xb->qs[il*8 + i] & 0x70;+ // float16 values+ ushort x0 = (em0 << (dst_m_bits - 1)) | ((xb->qs[il*8 + i] & 0x08) << 12);+ ushort x1 = (em1 << (dst_m_bits - 5)) | ((xb->qs[il*8 + i] & 0x80) << 8);++ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0 = x0 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1 = x1 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0 = dst_0p5 | (x0 & 0x8000);+ }+ if (em1 == 0x10) {+ x1 = dst_0p5 | (x1 & 0x8000);+ }+ // x is zero, do nothing++ if (isnan(scale)) {+ reg_f[i/2][2*(i%2) + 0] = scale;+ reg_f[i/2][2*(i%2) + 1] = scale;+ } else {+ reg_f[i/2][2*(i%2) + 0] = scale * as_type<half>(x0);+ reg_f[i/2][2*(i%2) + 1] = scale * as_type<half>(x1);+ }+ }+ reg = (type4x4) reg_f;+}+#define QK_NL 16//@@ -9289,6 +9342,8 @@ template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_mtemplate [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;+template [[host_name("kernel_mul_mm_mxfp4_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4>;+//// indirect matrix-matrix multiplication//@@ -9320,6 +9375,8 @@ template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_mtemplate [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;+template [[host_name("kernel_mul_mm_id_mxfp4_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4>;+//// matrix-vector multiplication@@ -9436,6 +9493,120 @@ kernel void kernel_mul_mv_id(sgitg);}+// MXFP32 implementation derived from mul_vec_q_n_f32_impl and block_q_n_dot_y+void mul_mv_mxfp4_f32_impl(+ ggml_metal_kargs_mul_mv args,+ device const char * src0,+ device const char * src1,+ device char * dst,+ threadgroup char * shmem,+ uint3 tgpig,+ ushort tiisg,+ ushort sgitg) {+ const ushort dst_bias = 15;+ const ushort dst_0p5 = 0x3800;+ const ushort dst_m_bits = 10;+ const int nr0 = N_R0_MXFP4;+ const int nsg = N_SG_MXFP4;+ const int nw = N_SIMDWIDTH;+ const int nb = args.ne00/MXFP4;++ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * nsg + sgitg) * nr0;++ const uint i12 = im%args.ne12;+ const uint i13 = im/args.ne12;++ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;++ device const float * y = (device const float *) (src1 + offset1);++ // pointers to src0 rows+ device const block_mxfp4 * ax[nr0];+ for (int row = 0; row < nr0; ++row) {+ const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;++ ax[row] = (device const block_mxfp4 *) ((device char *) src0 + offset0);+ }++ float yl[16]; // src1 vector cache+ float sumf[nr0] = {0.f};++ const short ix = (tiisg/2);+ const short il = (tiisg%2)*16;++ device const float * yb = y + ix*MXFP4 + il;++ // each thread in a SIMD group deals with half a block.+ for (int ib = ix; ib < nb; ib += nw/2) {++#pragma unroll+ for (short row = 0; row < nr0; row++) {+ // Processes 16 items+ device const block_mxfp4 * qb_curr = ax[row] + ib;+ float d = as_type<float>(((uint32_t)(ax[row] + ib)->d) << 23);+ // il = 0 or 16+ device const uint8_t *qs = ((device const uint8_t *) qb_curr + 1 + il/2);+ for (int i = 0; i < 8; ++i) {+ ushort em0 = qs[i] & 0x07;+ ushort em1 = qs[i] & 0x70;+ ushort x0 = (em0 << (dst_m_bits - 1)) | ((qs[i] & 0x08) << 12);+ ushort x1 = (em1 << (dst_m_bits - 5)) | ((qs[i] & 0x80) << 8);+ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0 = x0 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1 = x1 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0 = dst_0p5 | (x0 & 0x8000);+ }+ if (em1 == 0x10) {+ x1 = dst_0p5 | (x1 & 0x8000);+ }+ // x is zero, do nothing+ if (!isnan(d)) {+ sumf[row] += yb[i*2] * as_type<half>(x0) * d+ + yb[i*2+1] * as_type<half>(x1) * d;+ } else {+ sumf[row] = d;+ }+ }+ }++ yb += MXFP4 * 16;+ }++ device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0;++ for (int row = 0; row < nr0; ++row) {+ const float tot = simd_sum(sumf[row]);++ if (tiisg == 0 && first_row + row < args.ne01) {+ dst_f32[first_row + row] = tot;+ }+ }+}++[[host_name("kernel_mul_mv_mxfp4_f32")]]+kernel void kernel_mul_mv_mxfp4_f32(+ constant ggml_metal_kargs_mul_mv & args,+ device const char * src0,+ device const char * src1,+ device char * dst,+ threadgroup char * shmem [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ ushort tiisg[[thread_index_in_simdgroup]],+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {+ mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);+}+typedef decltype(kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>) kernel_mul_mv_id_t;template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>;@@ -9465,6 +9636,8 @@ template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_ttemplate [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL, N_SG_IQ4_NL, N_SIMDWIDTH>>>;template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS, N_SG_IQ4_XS, N_SIMDWIDTH>>>;+template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_mv_mxfp4_f32_impl>>;+kernel void kernel_pool_2d_max_f32(device const float * src0,device float * dst,diff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-impl.h b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-impl.hindex 17eab976f..938386ba8 100644--- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-impl.h@@ -65,6 +65,9 @@#define N_R0_IQ4_XS 2#define N_SG_IQ4_XS 2+#define N_R0_MXFP4 4+#define N_SG_MXFP4 2+// kernel argument structs//// - element counters (e.g. ne00) typically use int32_t to reduce register usagediff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.mindex ab46f6e3a..d8e05a21b 100644--- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m+++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m@@ -40,6 +40,7 @@ static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;static struct ggml_backend_reg g_ggml_backend_metal_reg;static struct ggml_backend_device g_ggml_backend_metal_device;+// information about a Metal device// note: assumes single GPU device - the default one// TODO: support multiple GPU devices@@ -209,6 +210,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_MXFP4_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_2,GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_3,GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_4,@@ -288,6 +290,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_MXFP4_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_BF16_F32,@@ -310,6 +313,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_MXFP4_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP0_F16,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP1_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F16,@@ -334,6 +338,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MXFP4_F16,GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32,GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16,GGML_METAL_KERNEL_TYPE_ROPE_MULTI_F32,@@ -934,7 +939,7 @@ static id<MTLLibrary> ggml_metal_load_library(id<MTLDevice> device, bool use_bflMTLCompileOptions * options = [MTLCompileOptions new];options.preprocessorMacros = prep;-+//[options setFastMathEnabled:false];metal_library = [device newLibraryWithSource:src options:options error:&error];@@ -1157,6 +1162,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t deGGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_MXFP4_F32, mul_mv_mxfp4_f32, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_2, mul_mv_ext_f16_f32_r1_2, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_3, mul_mv_ext_f16_f32_r1_3, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_4, mul_mv_ext_f16_f32_r1_4, has_simdgroup_reduction);@@ -1236,6 +1242,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t deGGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, has_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_MXFP4_F32, mul_mv_id_mxfp4_f32, has_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_BF16_F32, mul_mm_bf16_f32, has_simdgroup_mm && use_bfloat);@@ -1258,6 +1265,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t deGGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_MXFP4_F32, mul_mm_mxfp4_f32, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP0_F16, mul_mm_id_map0_f16, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP1_F32, mul_mm_id_map1_f32, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F16, mul_mm_id_f32_f16, has_simdgroup_mm);@@ -1282,6 +1290,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t deGGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16, mul_mm_id_iq1_m_f16, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16, mul_mm_id_iq4_nl_f16, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16, mul_mm_id_iq4_xs_f16, has_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MXFP4_F16, mul_mm_id_mxfp4_f16, has_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32, rope_norm_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16, rope_norm_f16, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_MULTI_F32, rope_multi_f32, true);@@ -3007,6 +3016,7 @@ static bool ggml_metal_encode_node(case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32 ].pipeline; break;case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break;case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32 ].pipeline; break;+ case GGML_TYPE_MXFP4: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_MXFP4_F32 ].pipeline; break;default: GGML_ABORT("MUL MAT-MAT not implemented");}@@ -3212,6 +3222,12 @@ static bool ggml_metal_encode_node(smem = 32*sizeof(float);pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32].pipeline;} break;+ case GGML_TYPE_MXFP4:+ {+ nsg = N_SG_MXFP4;+ nr0 = N_R0_MXFP4;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_MXFP4_F32].pipeline;+ } break;default:{GGML_LOG_ERROR("Asserting on type %d\n", (int)src0t);@@ -3396,6 +3412,7 @@ static bool ggml_metal_encode_node(case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16 ].pipeline; break;case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16 ].pipeline; break;case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16 ].pipeline; break;+ case GGML_TYPE_MXFP4: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MXFP4_F16 ].pipeline; break;default: GGML_ABORT("MUL_MAT_ID not implemented");}@@ -3607,6 +3624,12 @@ static bool ggml_metal_encode_node(smem = 32*sizeof(float);pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32].pipeline;} break;+ case GGML_TYPE_MXFP4:+ {+ nsg = N_SG_MXFP4;+ nr0 = N_R0_MXFP4;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_MXFP4_F32].pipeline;+ } break;default:{GGML_LOG_ERROR("Asserting on type %d\n", (int)src2t);diff --git a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.metal b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.metalindex 08e8d8070..69fa17de3 100644--- a/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.metal+++ b/ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.metal@@ -1902,16 +1902,16 @@ void mul_vec_q_n_f32_impl(device const char * src1,device char * dst,threadgroup char * shmem,- uint3 tgpig,- ushort tiisg,- ushort sgitg) {- const int nb = args.ne00/QK4_0;+ uint3 tgpig, // Threadgroup Position in Grid+ ushort tiisg, // Thread Index in SIMD Group+ ushort sgitg) { // SIMD Group Index in ThreadGroup+ const int nb = args.ne00/QK4_0; // src0->ne[0] / 32const int r0 = tgpig.x;const int r1 = tgpig.y;const int im = tgpig.z;- const int first_row = (r0 * nsg + sgitg) * nr0;+ const int first_row = (r0 * nsg + sgitg) * nr0; // nsg=2 nr0=4const uint i12 = im%args.ne12;const uint i13 = im/args.ne12;@@ -6744,6 +6744,49 @@ kernel void kernel_mul_mm_id(}}+template <typename type4x4>+void dequantize_mxfp4(device const block_mxfp4 * xb, short il, thread type4x4 & reg) {+ float4x4 reg_f;+ const ushort dst_bias = 15;+ const ushort dst_0p5 = 0x3800;+ const ushort dst_m_bits = 10;+ const half scale = (half)(as_type<float>(((uint32_t)xb->d) << 23));+ // il:0 first 16, il:1 last 16+ for (int i = 0; i < 8; i++) {+ ushort em0 = xb->qs[il*8 + i] & 0x07;+ ushort em1 = xb->qs[il*8 + i] & 0x70;+ // float16 values+ ushort x0 = (em0 << (dst_m_bits - 1)) | ((xb->qs[il*8 + i] & 0x08) << 12);+ ushort x1 = (em1 << (dst_m_bits - 5)) | ((xb->qs[il*8 + i] & 0x80) << 8);++ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0 = x0 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1 = x1 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0 = dst_0p5 | (x0 & 0x8000);+ }+ if (em1 == 0x10) {+ x1 = dst_0p5 | (x1 & 0x8000);+ }+ // x is zero, do nothing++ if (isnan(scale)) {+ reg_f[i/2][2*(i%2) + 0] = scale;+ reg_f[i/2][2*(i%2) + 1] = scale;+ } else {+ reg_f[i/2][2*(i%2) + 0] = scale * as_type<half>(x0);+ reg_f[i/2][2*(i%2) + 1] = scale * as_type<half>(x1);+ }+ }+ reg = (type4x4) reg_f;+}+#define QK_NL 16//@@ -6811,6 +6854,8 @@ template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_mtemplate [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;+template [[host_name("kernel_mul_mm_mxfp4_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4>;+//// indirect matrix-matrix multiplication//@@ -6842,6 +6887,8 @@ template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_mtemplate [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl>;template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs>;+template [[host_name("kernel_mul_mm_id_mxfp4_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4>;+//// matrix-vector multiplication@@ -6958,6 +7005,120 @@ kernel void kernel_mul_mv_id(sgitg);}+// MXFP32 implementation derived from mul_vec_q_n_f32_impl and block_q_n_dot_y+void mul_mv_mxfp4_f32_impl(+ ggml_metal_kargs_mul_mv args,+ device const char * src0,+ device const char * src1,+ device char * dst,+ threadgroup char * shmem,+ uint3 tgpig,+ ushort tiisg,+ ushort sgitg) {+ const ushort dst_bias = 15;+ const ushort dst_0p5 = 0x3800;+ const ushort dst_m_bits = 10;+ const int nr0 = N_R0_MXFP4;+ const int nsg = N_SG_MXFP4;+ const int nw = N_SIMDWIDTH;+ const int nb = args.ne00/MXFP4;++ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * nsg + sgitg) * nr0;++ const uint i12 = im%args.ne12;+ const uint i13 = im/args.ne12;++ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;++ device const float * y = (device const float *) (src1 + offset1);++ // pointers to src0 rows+ device const block_mxfp4 * ax[nr0];+ for (int row = 0; row < nr0; ++row) {+ const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;++ ax[row] = (device const block_mxfp4 *) ((device char *) src0 + offset0);+ }++ float yl[16]; // src1 vector cache+ float sumf[nr0] = {0.f};++ const short ix = (tiisg/2);+ const short il = (tiisg%2)*16;++ device const float * yb = y + ix*MXFP4 + il;++ // each thread in a SIMD group deals with half a block.+ for (int ib = ix; ib < nb; ib += nw/2) {++#pragma unroll+ for (short row = 0; row < nr0; row++) {+ // Processes 16 items+ device const block_mxfp4 * qb_curr = ax[row] + ib;+ float d = as_type<float>(((uint32_t)(ax[row] + ib)->d) << 23);+ // il = 0 or 16+ device const uint8_t *qs = ((device const uint8_t *) qb_curr + 1 + il/2);+ for (int i = 0; i < 8; ++i) {+ ushort em0 = qs[i] & 0x07;+ ushort em1 = qs[i] & 0x70;+ ushort x0 = (em0 << (dst_m_bits - 1)) | ((qs[i] & 0x08) << 12);+ ushort x1 = (em1 << (dst_m_bits - 5)) | ((qs[i] & 0x80) << 8);+ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0 = x0 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1 = x1 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0 = dst_0p5 | (x0 & 0x8000);+ }+ if (em1 == 0x10) {+ x1 = dst_0p5 | (x1 & 0x8000);+ }+ // x is zero, do nothing+ if (!isnan(d)) {+ sumf[row] += yb[i*2] * as_type<half>(x0) * d+ + yb[i*2+1] * as_type<half>(x1) * d;+ } else {+ sumf[row] = d;+ }+ }+ }++ yb += MXFP4 * 16;+ }++ device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0;++ for (int row = 0; row < nr0; ++row) {+ const float tot = simd_sum(sumf[row]);++ if (tiisg == 0 && first_row + row < args.ne01) {+ dst_f32[first_row + row] = tot;+ }+ }+}++[[host_name("kernel_mul_mv_mxfp4_f32")]]+kernel void kernel_mul_mv_mxfp4_f32(+ constant ggml_metal_kargs_mul_mv & args,+ device const char * src0,+ device const char * src1,+ device char * dst,+ threadgroup char * shmem [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ ushort tiisg[[thread_index_in_simdgroup]],+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {+ mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);+}+typedef decltype(kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>) kernel_mul_mv_id_t;template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_impl<float, float4, float, float4>>>;@@ -6987,6 +7148,8 @@ template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_ttemplate [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL, N_SG_IQ4_NL, N_SIMDWIDTH>>>;template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS, N_SG_IQ4_XS, N_SIMDWIDTH>>>;+template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_mv_mxfp4_f32_impl>>;+kernel void kernel_pool_2d_max_f32(device const float * src0,device float * dst,diff --git a/ml/backend/ggml/ggml/src/ggml-quants.c b/ml/backend/ggml/ggml/src/ggml-quants.cindex 84ec6dfe3..17c308aae 100644--- a/ml/backend/ggml/ggml/src/ggml-quants.c+++ b/ml/backend/ggml/ggml/src/ggml-quants.c@@ -4925,6 +4925,144 @@ void quantize_row_iq2_s_ref(const float * GGML_RESTRICT x, block_iq2_s * GGML_REquantize_iq2_s(x, y, 1, k, NULL);}+// =============================== mxfp4 (de)-quantization++void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k) {+ static const int qk = MXFP4;+ static const uint32_t E8_BIAS = 127;+ static const uint32_t E2_BIAS = 1;++ assert(k % qk == 0);++ const int nb = k / qk;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < qk; j++) {+ const float v = x[i*qk + j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ }+ }++ const float dequant_scale = amax / 6.0f;+ uint32_t dequant_scale_exponent = 0;+ memcpy(&dequant_scale_exponent, &dequant_scale, sizeof(dequant_scale_exponent));++ // Rounding up+ dequant_scale_exponent = (dequant_scale_exponent + 0x007FFFFF) & 0x7F800000;+ // Rounding down+ // dequant_scale_exponent = dequant_scale_exponent & 0x7F800000;++ float dequant_scale_rounded = 0.0f;+ memcpy(&dequant_scale_rounded, &dequant_scale_exponent, sizeof(dequant_scale_rounded));+ float quant_scale = 0.0f;+ if (dequant_scale_rounded != 0.0f) {+ quant_scale = 1.0f / dequant_scale_rounded;+ }++ y[i].d = (uint8_t)(dequant_scale_exponent >> 23);++ for (int j = 0; j < qk/2; ++j) {+ const float x0 = x[i*qk + j*2]*quant_scale;+ const float x1 = x[i*qk + j*2+1]*quant_scale;++ uint32_t xi0 = 0;+ uint32_t xi1 = 0;+ memcpy(&xi0, &x0, sizeof(xi0));+ memcpy(&xi1, &x1, sizeof(xi1));++ uint32_t s0 = xi0 & 0x80000000;+ uint32_t s1 = xi1 & 0x80000000;+ uint32_t e0 = (xi0 >> 23) & 0xFF;+ uint32_t e1 = (xi1 >> 23) & 0xFF;+ uint32_t m0 = (xi0 & 0x7FFFFF);+ uint32_t m1 = (xi1 & 0x7FFFFF);++ // 0.25 <= x < 0.75 maps to 0.5, a denormal number+ // Move implicit bit 1 at the beginning to mantissa for denormals+ // adjusted_exponents+ uint32_t ae0 = E8_BIAS - (e0 + 1);+ uint32_t ae1 = E8_BIAS - (e1 + 1);+ if (e0 < E8_BIAS) {+ m0 = (0x400000 | (m0 >> 1)) >> ae0;+ }+ if (e1 < E8_BIAS) {+ m1 = (0x400000 | (m1 >> 1)) >> ae1;+ }++ // For normal numbers, we change the bias from 127 to 1, and for subnormals, we keep exponent as 0.+ e0 = MAX(e0, E8_BIAS - E2_BIAS) - (E8_BIAS - E2_BIAS);+ e1 = MAX(e1, E8_BIAS - E2_BIAS) - (E8_BIAS - E2_BIAS);++ // Combine sign, exponent, and mantissa, while saturating+ // rounding nearest with tie breaking up by adding +1 to one bit right of the LSB, then shift right+ uint32_t tmp0 = MIN((((e0 << 2) | (m0 >> 21)) + 1) >> 1, 0x7);+ uint32_t tmp1 = MIN((((e1 << 2) | (m1 >> 21)) + 1) >> 1, 0x7);+ uint8_t v0 = (uint8_t)((s0 >> 28) | tmp0);+ uint8_t v1 = (uint8_t)((s1 >> 28) | tmp1);+ y[i].qs[j] = v0;+ y[i].qs[j] |= v1 << 4;+ }+ }+}++void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ assert(k % MXFP4 == 0);++ const int nb = k / MXFP4;+ const uint16_t dst_bias = 15;+ const uint16_t dst_0p5 = 0x3800;+ const uint16_t dst_m_bits = 10;++ for (int i = 0; i < nb; i++) {+ union {+ uint32_t as_bits;+ float as_value;+ } scale;+ scale.as_bits = (((uint32_t)x[i].d) << 23);+ for (int j = 0; j < MXFP4/2; ++j) {+ uint16_t em0 = x[i].qs[j] & 0x07;+ uint16_t em1 = x[i].qs[j] & 0x70;+ // float16 values+ uint16_t x0 = (em0 << (dst_m_bits - 1)) | ((x[i].qs[j] & 0x08) << 12);+ uint16_t x1 = (em1 << (dst_m_bits - 5)) | ((x[i].qs[j] & 0x80) << 8);++ // Three cases:+ // x is normal and non-zero: Correct bias+ if ((em0 & 0x06) != 0) {+ x0 = x0 + ((dst_bias - 1) << dst_m_bits);+ }+ if ((em1 & 0x60) != 0) {+ x1 = x1 + ((dst_bias - 1) << dst_m_bits);+ }+ // x is subnormal (x == 0bs001 where s is the sign): Map to +-0.5 in the dst type+ if (em0 == 0x01) {+ x0 = dst_0p5 | (x0 & 0x8000);+ }+ if (em1 == 0x10) {+ x1 = dst_0p5 | (x1 & 0x8000);+ }+ // x is zero, do nothing++ if (isnan(scale.as_value)) {+ y[i*MXFP4 + j*2] = scale.as_value;+ y[i*MXFP4 + j*2+1] = scale.as_value;+ } else {+ y[i*MXFP4 + j*2] = GGML_FP16_TO_FP32(x0)*scale.as_value;+ y[i*MXFP4 + j*2+1] = GGML_FP16_TO_FP32(x1)*scale.as_value;+ }+ }+ }+}+++size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {+ quantize_row_mxfp4_ref(src, dst, (int64_t)nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row);+}+// =============================== data validationstatic bool validate_float(float f, size_t i) {@@ -5214,7 +5352,9 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte{VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);} break;-+ case GGML_TYPE_MXFP4:+ // TODO - anything to validate?+ break;case GGML_TYPE_I8:case GGML_TYPE_I16:case GGML_TYPE_I32:diff --git a/ml/backend/ggml/ggml/src/ggml-quants.h b/ml/backend/ggml/ggml/src/ggml-quants.hindex d09173e11..2fc40f754 100644--- a/ml/backend/ggml/ggml/src/ggml-quants.h+++ b/ml/backend/ggml/ggml/src/ggml-quants.h@@ -37,6 +37,8 @@ GGML_API void quantize_row_iq4_xs_ref (const float * GGML_RESTRICT x, block_iq4_GGML_API void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);+GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k);+// DequantizationGGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);@@ -65,6 +67,8 @@ GGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, floaGGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);GGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")GGML_API size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);@@ -90,6 +94,8 @@ GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRGGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+GGML_API void iq2xs_init_impl(enum ggml_type type);GGML_API void iq2xs_free_impl(enum ggml_type type);GGML_API void iq3xs_init_impl(int grid_size);diff --git a/ml/backend/ggml/ggml/src/ggml.c b/ml/backend/ggml/ggml/src/ggml.cindex 8a6546240..0f3c98340 100644--- a/ml/backend/ggml/ggml/src/ggml.c+++ b/ml/backend/ggml/ggml/src/ggml.c@@ -589,11 +589,13 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {.to_float = (ggml_to_float_t) dequantize_row_q4_1,.from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref,},- [4] = { // GGML_TYPE_Q4_2- .type_name = "DEPRECATED",- .blck_size = 0,- .type_size = 0,- .is_quantized = false,+ [GGML_TYPE_MXFP4] = { // formerly deprecated GGML_TYPE_Q4_2+ .type_name = "mxfp4",+ .blck_size = MXFP4,+ .type_size = sizeof(block_mxfp4),+ .is_quantized = true,+ .to_float = (ggml_to_float_t) dequantize_row_mxfp4,+ .from_float_ref = (ggml_from_float_t) quantize_row_mxfp4_ref,},[5] = { // GGML_TYPE_Q4_3.type_name = "DEPRECATED",@@ -6446,6 +6448,7 @@ size_t ggml_quantize_chunk(case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;case GGML_TYPE_F16:{size_t elemsize = sizeof(ggml_fp16_t);diff --git a/ml/backend/ggml/ggml_test.go b/ml/backend/ggml/ggml_test.gonew file mode 100644index 000000000..70ebb9df4--- /dev/null+++ b/ml/backend/ggml/ggml_test.go@@ -0,0 +1,60 @@+package ggml++import (+ "bytes"+ "log/slog"+ "os"+ "slices"+ "testing"++ "github.com/ollama/ollama/envconfig"+ "github.com/ollama/ollama/fs/ggml"+ "github.com/ollama/ollama/logutil"+ "github.com/ollama/ollama/ml"+)++func TestMain(m *testing.M) {+ slog.SetDefault(logutil.NewLogger(os.Stderr, envconfig.LogLevel()))+ os.Exit(m.Run())+}++func setup(tb testing.TB) ml.Backend {+ tb.Helper()++ f, err := os.CreateTemp(tb.TempDir(), "*.bin")+ if err != nil {+ tb.Fatal(err)+ }+ defer f.Close()++ if err := ggml.WriteGGUF(f, ggml.KV{+ "general.architecture": "test",+ "test.block_count": uint32(1),+ }, []*ggml.Tensor{+ {Name: "blk.0.weight", Shape: []uint64{1}, WriterTo: bytes.NewBuffer(slices.Repeat([]byte{0}, 4))},+ }); err != nil {+ tb.Fatal(err)+ }++ b, err := New(f.Name(), ml.BackendParams{NumGPULayers: 1})+ if err != nil {+ tb.Fatal(err)+ }++ return b+}++// initContextOrSkip takes a testing.T and true for GPU+// If GPUs are not available, the current test is skipped+// gpu=false will always succed+func initContextOrSkip(t *testing.T, b ml.Backend, gpu bool) ml.Context {+ if gpu && len(b.(*Backend).schedBackends) == 1 {+ t.Skip("No GPU detected, skipping GPU test case")+ }+ ctx := b.NewContext()+ t.Cleanup(func() { ctx.Close() })+ if gpu {+ return ctx.Layer(0)+ }+ return ctx.Input()+}diff --git a/ml/backend/ggml/mxfp4_test.go b/ml/backend/ggml/mxfp4_test.gonew file mode 100644index 000000000..3c17eb8aa--- /dev/null+++ b/ml/backend/ggml/mxfp4_test.go@@ -0,0 +1,795 @@+package ggml++import (+ "math"+ "math/rand"+ "os"+ "testing"++ "github.com/ollama/ollama/ml"++ fsggml "github.com/ollama/ollama/fs/ggml"+)++/*+ To get GPUs loading in these tests on windows...++ $env:OLLAMA_LIBRARY_PATH="$(pwd)\build\lib\ollama"+ $env:PATH="$(pwd)\build\lib\ollama;$env:PATH"++ go test .\ml\backend\ggml\... -run TestMXFP4+*/++// MXFP4 reference: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf++// E2M1 values+var mxfp4_vals = []float32{+ 0.0, // 0 00 0 = 0x0+ 0.5, // 0 00 1 = 0x1+ 1.0, // 0 01 0 = 0x2+ 1.5, // 0 01 1 = 0x3+ 2.0, // 0 10 0 = 0x4+ 3.0, // 0 10 1 = 0x5+ 4.0, // 0 11 0 = 0x6+ 6.0, // 0 11 1 = 0x7+ 0.0, // 1 00 0 = 0x8+ -0.5, // 1 00 1 = 0x9+ -1.0, // 1 01 0 = 0xa+ -1.5, // 1 01 1 = 0xb+ -2.0, // 1 10 0 = 0xc+ -3.0, // 1 10 1 = 0xd+ -4.0, // 1 11 0 = 0xe+ -6.0, // 1 11 1 = 0xf+}++func TestMXFP4Ops(t *testing.T) {+ b := setup(t)+ for _, useGPU := range []bool{false, true} {+ useGPU := useGPU+ var label string+ if useGPU {+ label = "gpu"+ } else {+ label = "cpu"+ }+ t.Run(label, func(t *testing.T) {+ t.Run("mulmatid", func(t *testing.T) {+ // Use exact values that are supported without scaling so we can compare against an fp32 tensor+ t.Run("exact", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s00 = 64+ const s01 = 1+ const s02 = 2+ const s10 = s00+ const s11 = 1+ const s12 = 1+ // const s00 = 2880+ // const s01 = 5760+ // const s02 = 32+ // const s10 = s00+ // const s11 = 1+ // const s12 = 64++ data := [s00 * s01 * s02]float32{}+ for i := range data {+ data[i] = mxfp4_vals[r.Int()%len(mxfp4_vals)]+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s00, s01, s02)+ t1f := ctx.(*Context).FromFloatSlice(data[:], s00, s01, s02)+ // for i := range len(data) / 32 { // MXFP4 block size+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", data[i*32+j])+ // }+ // t.Logf(" t1[%s]\n", strings.Join(vals[:], ", "))+ // }++ // random 0-1 float+ d2 := [s10 * s11 * s12]float32{}+ for i := range d2 {+ d2[i] = float32(r.Float32())+ }+ // for i := range len(d2) / s10 {+ // vals := [s10]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", d2[i*s10+j])+ // }+ // t.Logf(" t2[%s]\n", strings.Join(vals[:], ", "))+ // }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], s10, s11, s12)++ d3 := [4 * s12]int32{}+ for i := range d3 {+ d3[i] = int32(i) % s02+ }+ t3 := ctx.(*Context).FromIntSlice(d3[:], 4, s12)++ // t.Log("calling MulmatID")+ t4 := t1.MulmatID(ctx, t2, t3)+ t4f := t1f.MulmatID(ctx, t2, t3)+ d4 := ml.Dump(ctx, t4, ml.DumpWithPrecision(2)) // lower precision for CPU accuracy+ d4f := ml.Dump(ctx, t4f, ml.DumpWithPrecision(2))+ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ // t.Logf("MulmatID results matched:\n%s", d4)+ })++ t.Run("range", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s0 = 64+ const s1 = 2+ const s2 = 4+ const idlen = 4+ data := [s0 * s1 * s2]float32{}+ inTotal := float32(0)+ for i := range data {+ data[i] = float32(i)+ inTotal += float32(i)+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ // Reconvert back to floats to remove the quantization fidelity loss for comparison+ dataf := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s0, s1, s2)+ t1f := ctx.(*Context).FromFloatSlice(dataf, s0, s1, s2)+ // for i := range len(data) / 32 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", dataf[i*32+j])+ // }+ // t.Logf(" t1[%s]\n", strings.Join(vals[:], ", "))+ // }++ d2 := [s0]float32{}+ for i := range d2 {+ // d2[i] = float32(i)+ d2[i] = float32(r.Float32())+ }+ // for i := range len(d2) / s0 {+ // vals := [s0]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", d2[i*s0+j])+ // }+ // t.Logf(" t2[%s]\n", strings.Join(vals[:], ", "))+ // }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], s0)++ // TODO - there might be a CUDA bug here...+ d3 := [idlen]int32{1, 1, 2, 3}+ // for i := range d3 {+ // d3[i] = int32(i) % s2+ // t.Logf("%d] %d", i, d3[i])+ // }+ t3 := ctx.(*Context).FromIntSlice(d3[:], idlen)++ // t.Log("calling Mulmat")+ t4 := t1.MulmatID(ctx, t2, t3)+ t4f := t1f.MulmatID(ctx, t2, t3)+ // Metal has some drift so use reduced precision for dump comparisons+ d4 := ml.Dump(ctx, t4, ml.DumpWithPrecision(2))+ d4f := ml.Dump(ctx, t4f, ml.DumpWithPrecision(2))+ r4 := t4.Floats()+ r4f := t4f.Floats()+ sim := cosineSimilarity(r4, r4f)+ if sim < 0.99 {+ t.Logf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)++ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ // t.Logf("mxfp4 result\n%s", d4)+ })+ t.Run("random", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s00 = 2880+ const s01 = 5760+ const s02 = 32+ const s10 = s00+ const s11 = 1+ const s12 = 64+ const idlen = 4++ data := [s00 * s01 * s02]float32{}+ for i := range data {+ data[i] = float32(r.Float32() * 10.0)+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ // Reconvert back to floats to remove the quantization fidelity loss for comparison+ dataf := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s00, s01, s02)+ t1f := ctx.(*Context).FromFloatSlice(dataf, s00, s01, s02)+ // for i := range len(data) / 32 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", dataf[i*32+j])+ // }+ // t.Logf(" t1[%s]\n", strings.Join(vals[:], ", "))+ // }++ d2 := [s10 * s11 * s12]float32{}+ for i := range d2 {+ // d2[i] = float32(i)+ d2[i] = float32(r.Float32())+ }+ // for i := range len(d2) / s0 {+ // vals := [s0]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", d2[i*s0+j])+ // }+ // t.Logf(" t2[%s]\n", strings.Join(vals[:], ", "))+ // }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], s10, s11, s12)++ // arange equiv+ d3 := [idlen * s12]int32{}+ for i := range d3 {+ d3[i] = int32(i) % s02+ }+ t3 := ctx.(*Context).FromIntSlice(d3[:], idlen, s12)++ // t.Log("calling Mulmat")+ // t3 := t1.Mulmat(ctx, t2)+ // t3f := t1f.Mulmat(ctx, t2)+ t4 := t1.MulmatID(ctx, t2, t3)+ t4f := t1f.MulmatID(ctx, t2, t3)+ // Metal and CPU have some drift so use reduced precision for dump comparisons+ d4 := ml.Dump(ctx, t4, ml.DumpWithPrecision(1))+ d4f := ml.Dump(ctx, t4f, ml.DumpWithPrecision(1))+ // t.Logf("mxfp4 data: \n%s", d4)+ r4 := t4.Floats()+ r4f := t4f.Floats()+ sim := cosineSimilarity(r4, r4f)+ if sim < 0.99 {+ t.Logf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)++ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ })++ // Use data file(s) with real data+ t.Run("example_7", func(t *testing.T) {+ ctx := initContextOrSkip(t, b, useGPU)+ data0, err := os.ReadFile("mlp-gateup.bin")+ if err != nil {+ t.Skip("missing mlp-gateup.bin file, skipping test")+ }+ data1, err := os.ReadFile("hidden-states-7.bin")+ if err != nil {+ t.Skip("missing hidden-states.bin file, skipping test")+ }+ data2, err := os.ReadFile("selected-experts-7.bin")+ if err != nil {+ t.Skip("missing selected-experts.bin file, skipping test")+ }++ dtype := ml.DTypeMXFP4+ data0f := ConvertToF32(data0, uint32(fsggml.TensorTypeMXFP4), 2880*5760*32)+ t1 := ctx.(*Context).FromBytes(dtype, data0, 2880, 5760, 32)+ t1f := ctx.(*Context).FromFloatSlice(data0f, 2880, 5760, 32)++ // t.Logf("f32: \n%s", ml.Dump(ctx, t1f))++ t2 := ctx.(*Context).FromBytes(ml.DTypeF32, data1, 2880, 1, 7)+ // t.Logf("hidden-state: \n%s", ml.Dump(ctx, t2))++ t3 := ctx.(*Context).FromBytes(ml.DTypeI32, data2, 4, 7)+ // t.Logf("experts: \n%s", ml.Dump(ctx, t3))++ // t.Log("calling MulmatID")+ t4 := t1.MulmatID(ctx, t2, t3)+ t4f := t1f.MulmatID(ctx, t2, t3)++ d4 := ml.Dump(ctx, t4)+ d4f := ml.Dump(ctx, t4f)++ r4 := t4.Floats()+ r4f := t4f.Floats()+ sim := cosineSimilarity(r4, r4f)+ if sim < 0.99 {+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)++ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ // t.Logf("MulmatID results matched:\n%s", d4)+ })++ // Use data file(s) with real data+ t.Run("example_384", func(t *testing.T) {+ ctx := initContextOrSkip(t, b, useGPU)+ data0, err := os.ReadFile("mlp-gateup.bin")+ if err != nil {+ t.Skip("missing mlp-gateup.bin file, skipping test")+ }+ data1, err := os.ReadFile("hidden-states-384.bin")+ if err != nil {+ t.Skip("missing hidden-states.bin file, skipping test")+ }+ data2, err := os.ReadFile("selected-experts-384.bin")+ if err != nil {+ t.Skip("missing selected-experts.bin file, skipping test")+ }++ dtype := ml.DTypeMXFP4+ data0f := ConvertToF32(data0, uint32(fsggml.TensorTypeMXFP4), 2880*5760*32)+ t1 := ctx.(*Context).FromBytes(dtype, data0, 2880, 5760, 32)+ t1f := ctx.(*Context).FromFloatSlice(data0f, 2880, 5760, 32)++ // t.Logf("f32: \n%s", ml.Dump(ctx, t1f))++ t2 := ctx.(*Context).FromBytes(ml.DTypeF32, data1, 2880, 1, 384)+ // t.Logf("hidden-state: \n%s", ml.Dump(ctx, t2))++ t3 := ctx.(*Context).FromBytes(ml.DTypeI32, data2, 4, 384)+ // t.Logf("experts: \n%s", ml.Dump(ctx, t3))++ // t.Log("calling MulmatID")+ t4 := t1.MulmatID(ctx, t2, t3)+ t4f := t1f.MulmatID(ctx, t2, t3)++ d4 := ml.Dump(ctx, t4, ml.DumpWithPrecision(3))+ d4f := ml.Dump(ctx, t4f, ml.DumpWithPrecision(3))++ r4 := t4.Floats()+ r4f := t4f.Floats()+ sim := cosineSimilarity(r4, r4f)+ if sim < 0.99 {+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)++ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ // t.Logf("MulmatID results matched:\n%s", d4)+ })++ // Use data file(s) with real data+ t.Run("example_1d", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ data0, err := os.ReadFile("mlp-gateup.bin")+ if err != nil {+ t.Skip("missing mlp-gateup.bin file, skipping test")+ }++ dtype := ml.DTypeMXFP4+ data0f := ConvertToF32(data0, uint32(fsggml.TensorTypeMXFP4), 2880*5760*32)+ t1 := ctx.(*Context).FromBytes(dtype, data0, 2880, 5760, 32)+ t1f := ctx.(*Context).FromFloatSlice(data0f, 2880, 5760, 32)++ // t.Logf("f32: \n%s", ml.Dump(ctx, t1f))+ data1 := [2880]float32{}+ for i := range data1 {+ data1[i] = float32(r.Float32())+ }++ t2 := ctx.(*Context).FromFloatSlice(data1[:], 2880)+ // t.Logf("hidden-state: \n%s", ml.Dump(ctx, t2))+ data2 := [4]int32{+ 12, 30, 17, 7,+ // 7, 17, 12, 30,+ }++ t3 := ctx.(*Context).FromIntSlice(data2[:], 4)+ // t.Logf("experts: \n%s", ml.Dump(ctx, t3))++ // t.Log("calling MulmatID")+ t4 := t1.MulmatID(ctx, t2, t3)+ t4f := t1f.MulmatID(ctx, t2, t3)++ d4 := ml.Dump(ctx, t4)+ d4f := ml.Dump(ctx, t4f)++ r4 := t4.Floats()+ r4f := t4f.Floats()+ sim := cosineSimilarity(r4, r4f)+ if sim < 0.99 {+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)++ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ // t.Logf("MulmatID results matched:\n%s", d4)+ })+ })++ t.Run("mm", func(t *testing.T) {+ t.Run("example", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ data0, err := os.ReadFile("mlp-gateup.bin")+ if err != nil {+ t.Skip("missing mlp-gateup.bin file, skipping test")+ }+ data1 := [2880 * 1 * 32]float32{}+ for i := range data1 {+ data1[i] = float32(r.Float32())+ }++ dtype := ml.DTypeMXFP4+ data0f := ConvertToF32(data0, uint32(fsggml.TensorTypeMXFP4), 2880*5760*32)+ t1 := ctx.(*Context).FromBytes(dtype, data0, 2880, 5760, 32)+ t1f := ctx.(*Context).FromFloatSlice(data0f, 2880, 5760, 32)++ // t.Logf("f32: \n%s", ml.Dump(ctx, t1f))++ t2 := ctx.(*Context).FromFloatSlice(data1[:], 2880, 1, 32)++ t4 := t1.Mulmat(ctx, t2)+ t4f := t1f.Mulmat(ctx, t2)++ d4 := ml.Dump(ctx, t4, ml.DumpWithPrecision(3))+ d4f := ml.Dump(ctx, t4f, ml.DumpWithPrecision(3))++ r4 := t4.Floats()+ r4f := t4f.Floats()+ sim := cosineSimilarity(r4, r4f)+ if sim < 0.99 {+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)++ if d4 != d4f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d4f, d4)+ }+ // t.Logf("Mulmat results matched:\n%s", d4)+ })++ t.Run("exact/3x3", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s10 = 64+ const s11 = 1+ const s12 = 2+ const s20 = s10+ const s21 = 1+ const s22 = 2++ data := [s10 * s11 * s12]float32{}+ for i := range data {+ data[i] = mxfp4_vals[r.Int()%len(mxfp4_vals)]+ }+ // for i := range len(data) / 32 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", data[i*32+j])+ // }+ // t.Logf(" [%s]\n", strings.Join(vals[:], ", "))+ // }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ // for i := range len(mxData) / 17 {+ // vals := [17]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2x", mxData[i*17+j])+ // }+ // t.Logf(" %s\n", strings.Join(vals[:], ", "))+ // }+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s10, s11, s12)+ t1f := ctx.(*Context).FromFloatSlice(data[:], s10, s11, s12)++ d2 := [s20 * s21 * s22]float32{}+ for i := range d2 {+ d2[i] = float32(r.Float32())+ }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], s20, s21, s22)++ t3f := t1f.Mulmat(ctx, t2)+ t3 := t1.Mulmat(ctx, t2)+ d3 := ml.Dump(ctx, t3)+ d3f := ml.Dump(ctx, t3f)+ if d3 != d3f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ }+ })++ t.Run("exact/2x2", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s0 = 32+ const s1 = 64++ data := [s0 * s1]float32{}+ for i := range data {+ data[i] = mxfp4_vals[r.Int()%len(mxfp4_vals)]+ }+ // for i := range 4 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", data[i*32+j])+ // }+ // t.Logf(" [%s]\n", strings.Join(vals[:], ", "))+ // }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ // for i := range len(mxData) / 17 {+ // vals := [17]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2x", mxData[i*17+j])+ // }+ // t.Logf(" %s\n", strings.Join(vals[:], ", "))+ // }+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s0, s1)+ t1f := ctx.(*Context).FromFloatSlice(data[:], s0, s1)++ d2 := [s0 * s1]float32{}+ for i := range d2 {+ d2[i] = float32(r.Float32())+ }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], s0, s1)++ t3f := t1f.Mulmat(ctx, t2)+ t3 := t1.Mulmat(ctx, t2)+ d3 := ml.Dump(ctx, t3)+ d3f := ml.Dump(ctx, t3f)+ if d3 != d3f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ }+ })+ t.Run("exact/2x1", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s0 = 64+ const s1 = 4++ data := [s0 * s1]float32{}+ for i := range data {+ data[i] = mxfp4_vals[r.Int()%len(mxfp4_vals)]+ }+ // for i := range len(data) / 32 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", data[i*32+j])+ // }+ // t.Logf(" t1[%s]\n", strings.Join(vals[:], ", "))+ // }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ // for i := range len(mxData) / 17 {+ // vals := [17]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2x", mxData[i*17+j])+ // }+ // t.Logf(" %s\n", strings.Join(vals[:], ", "))+ // }+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s0, s1)+ t1f := ctx.(*Context).FromFloatSlice(data[:], s0, s1)++ d2 := [s0]float32{}+ for i := range d2 {+ d2[i] = float32(r.Float32())+ }+ // for i := range len(d2) / 32 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", d2[i*32+j])+ // }+ // t.Logf(" t2[%s]\n", strings.Join(vals[:], ", "))+ // }++ t2 := ctx.(*Context).FromFloatSlice(d2[:], s0)++ t3f := t1f.Mulmat(ctx, t2)+ t3 := t1.Mulmat(ctx, t2)+ d3 := ml.Dump(ctx, t3, ml.DumpWithPrecision(3))+ d3f := ml.Dump(ctx, t3f, ml.DumpWithPrecision(3))+ if d3 != d3f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ }+ })++ t.Run("range/2d", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))+ ctx := initContextOrSkip(t, b, useGPU)+ const s0 = 32+ const s1 = 4+ data := [s0 * s1]float32{}+ inTotal := float32(0)+ for i := range data {+ data[i] = float32(i)+ inTotal += float32(i)+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ // Reconvert back to floats to remove the quantization fidelity loss for comparison+ dataf := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))+ dtype := ml.DTypeMXFP4+ t1 := ctx.(*Context).FromBytes(dtype, mxData, s0, s1)+ t1f := ctx.(*Context).FromFloatSlice(dataf, s0, s1)+ // for i := range len(data) / 32 {+ // vals := [32]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", dataf[i*32+j])+ // }+ // t.Logf(" t1[%s]\n", strings.Join(vals[:], ", "))+ // }++ d2 := [s0 * s1]float32{}+ for i := range d2 {+ // d2[i] = float32(i)+ d2[i] = float32(r.Float32())+ }+ // for i := range len(d2) / s0 {+ // vals := [s0]string{}+ // for j := range vals {+ // vals[j] = fmt.Sprintf("%0.2f", d2[i*s0+j])+ // }+ // t.Logf(" t2[%s]\n", strings.Join(vals[:], ", "))+ // }++ t2 := ctx.(*Context).FromFloatSlice(d2[:], s0, s1)++ // t.Log("calling Mulmat")+ t3 := t1.Mulmat(ctx, t2)+ t3f := t1f.Mulmat(ctx, t2)+ d3 := ml.Dump(ctx, t3, ml.DumpWithPrecision(2))+ d3f := ml.Dump(ctx, t3f, ml.DumpWithPrecision(2))+ r3 := t3.Floats()+ r3f := t3f.Floats()+ sim := cosineSimilarity(r3, r3f)+ if sim < 0.99 {+ t.Logf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)+ if d3 != d3f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ }+ })++ t.Run("range/3d", func(t *testing.T) {+ ctx := initContextOrSkip(t, b, useGPU)+ data := [32 * 4 * 2]float32{}+ inTotal := float32(0)+ for i := range data {+ data[i] = float32(i)+ inTotal += float32(i)+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ dtype := ml.DTypeMXFP4+ // Reconvert back to floats to remove the quantization fidelity loss for comparison+ dataf := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))+ t1 := ctx.(*Context).FromBytes(dtype, mxData, 32, 4, 2)+ t1f := ctx.(*Context).FromFloatSlice(dataf, 32, 4, 2)++ d2 := [32 * 4 * 2]float32{}+ for i := range d2 {+ d2[i] = 2.0+ }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], 32, 4, 2)++ // t.Log("calling Mulmat")+ t3 := t1.Mulmat(ctx, t2)+ t3f := t1f.Mulmat(ctx, t2)+ d3 := ml.Dump(ctx, t3)+ d3f := ml.Dump(ctx, t3f)+ r3 := t3.Floats()+ r3f := t3f.Floats()+ sim := cosineSimilarity(r3, r3f)+ if sim < 0.99 {+ t.Logf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)+ if d3 != d3f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ }+ })+ })+ })+ }+}++func TestMXFP4Simple(t *testing.T) {+ b := setup(t)++ t.Run("fixed", func(t *testing.T) {+ ctx := initContextOrSkip(t, b, false)+ data := [32 * 2]float32{+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ dtype := ml.DTypeMXFP4+ // Reconvert back to floats to remove the quantization fidelity loss for comparison+ dataf := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))+ t1 := ctx.(*Context).FromBytes(dtype, mxData, 32, 2)+ t1f := ctx.(*Context).FromFloatSlice(dataf, 32, 2)++ d2 := [32 * 2]float32{+ // 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+ }+ t2 := ctx.(*Context).FromFloatSlice(d2[:], 32, 2)++ t.Log("calling Mulmat")+ t3f := t1f.Mulmat(ctx, t2)+ t3 := t1.Mulmat(ctx, t2)+ d3 := ml.Dump(ctx, t3)+ d3f := ml.Dump(ctx, t3f)+ if d3 != d3f {+ t.Fatalf("expected (f32): \n%s\n\n but got (mxfp4): \n%s", d3f, d3)+ }+ t.Logf("result (mxfp4): \n%s", d3)+ })+}++func TestMXFP4Conversion(t *testing.T) {+ t.Run("quantize/exact", func(t *testing.T) {+ r := rand.New(rand.NewSource(0))++ data := [32 * 4]float32{}+ for i := range data {+ data[i] = mxfp4_vals[r.Int()%len(mxfp4_vals)]+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ newData := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))++ if len(data) != len(newData) {+ t.Fatalf("length mismatch. started with %d but got %d", len(data), len(newData))+ }+ for i := range data {+ if data[i] != newData[i] {+ t.Logf("started with: %v", data)+ t.Logf("got : %v", newData)+ t.Fatalf("mismatched data starting at offset %d started with %f but got %f", i, data[i], newData[i])+ }+ }+ })+ t.Run("quantize/arange", func(t *testing.T) {+ data := [32 * 8]float32{}+ for i := range data {+ data[i] = float32(i) // / float32(6.0)+ }+ mxData := Quantize(fsggml.TensorTypeMXFP4, data[:], []uint64{uint64(len(data))})+ newData := ConvertToF32(mxData, uint32(fsggml.TensorTypeMXFP4), uint64(len(data)))++ if len(data) != len(newData) {+ t.Fatalf("length mismatch. started with %d but got %d", len(data), len(newData))+ }+ sim := cosineSimilarity(data[:], newData)+ if sim < 0.99 {+ t.Fatalf("failed similarity test: %f", sim)+ }+ t.Logf("similarity: %f", sim)+ })+}++func dotProduct[V float32 | float64](v1, v2 []V) V {+ var result V = 0+ for i := range v1 {+ result += v1[i] * v2[i]+ }+ return result+}++func magnitude[V float32 | float64](v []V) V {+ var result V = 0+ for _, val := range v {+ result += val * val+ }+ return V(math.Sqrt(float64(result)))+}++func cosineSimilarity[V float32 | float64](v1, v2 []V) V {+ return dotProduct(v1, v2) / (magnitude(v1) * magnitude(v2))+}diff --git a/ml/backend/ggml/quantization.go b/ml/backend/ggml/quantization.goindex bb31e455d..648ab74bb 100644--- a/ml/backend/ggml/quantization.go+++ b/ml/backend/ggml/quantization.go@@ -44,6 +44,8 @@ func ConvertToF32(data []byte, dtype uint32, nelements uint64) []float32 {C.dequantize_row_q6_K((*C.block_q6_K)(unsafe.Pointer(&data[0])), (*C.float)(&f32s[0]), elems)case C.GGML_TYPE_BF16:C.ggml_bf16_to_fp32_row((*C.ggml_bf16_t)(unsafe.Pointer(&data[0])), (*C.float)(&f32s[0]), elems)+ case C.GGML_TYPE_MXFP4:+ C.dequantize_row_mxfp4((*C.block_mxfp4)(unsafe.Pointer(&data[0])), (*C.float)(&f32s[0]), elems)default:panic("unsupported quantization format")}diff --git a/ml/nn/linear.go b/ml/nn/linear.goindex 3985dd6c8..5bcde84de 100644--- a/ml/nn/linear.go+++ b/ml/nn/linear.go@@ -15,3 +15,26 @@ func (m *Linear) Forward(ctx ml.Context, t ml.Tensor) ml.Tensor {return t}++type LinearBatch struct {+ Weight ml.Tensor `gguf:"weight"`+ Bias ml.Tensor `gguf:"bias"`+}++func (m *LinearBatch) Forward(ctx ml.Context, t, indices ml.Tensor) ml.Tensor {+ t = m.Weight.MulmatID(ctx, t, indices)+ if m.Bias != nil {+ var bias ml.Tensor+ if len(indices.Shape()) > 1 {+ // FIXME: Rows does not support 2D indices for a 2D input tensor so reshape indices to 1D.+ bias = m.Bias.Rows(ctx, indices.Contiguous(ctx, indices.Dim(0)*indices.Dim(1))).+ Duplicate(ctx).+ Reshape(ctx, m.Bias.Dim(0), indices.Dim(0), indices.Dim(1))+ } else {+ bias = m.Bias.Rows(ctx, indices)+ }+ t = t.Add(ctx, bias)+ }++ return t+}diff --git a/ml/nn/rope/rope.go b/ml/nn/rope/rope.goindex b0c00a5b9..3b72d1cf9 100644--- a/ml/nn/rope/rope.go+++ b/ml/nn/rope/rope.go@@ -4,9 +4,15 @@ import "github.com/ollama/ollama/ml"// Options contains optional parameters for RoPE functiontype Options struct {- OriginalContextLength intType intFactors ml.Tensor+ OriginalContextLength int++ // YaRN options+ ExtrapolationFactor,+ AttentionFactor,+ BetaFast,+ BetaSlow float32}// WithOriginalContextLength sets a custom context length@@ -31,3 +37,15 @@ func WithFactors(factors ml.Tensor) func(*Options) {}}}++func WithExtrapolationFactor(extrapolationFactor float32) func(*Options) {+ return func(opts *Options) {+ opts.ExtrapolationFactor = extrapolationFactor+ }+}++func WithAttentionFactor(attentionFactor float32) func(*Options) {+ return func(opts *Options) {+ opts.AttentionFactor = attentionFactor+ }+}diff --git a/model/bytepairencoding.go b/model/bytepairencoding.goindex 246d2ba3e..7ade497da 100644--- a/model/bytepairencoding.go+++ b/model/bytepairencoding.go@@ -22,7 +22,7 @@ var _ TextProcessor = (*BytePairEncoding)(nil)func NewBytePairEncoding(pre string, vocab *Vocabulary) BytePairEncoding {return BytePairEncoding{- pre: regexp2.MustCompile(pre, regexp2.Unicode|regexp2.RE2),+ pre: regexp2.MustCompile(pre, regexp2.None),vocab: vocab,}}diff --git a/model/models/gptoss/model.go b/model/models/gptoss/model.gonew file mode 100644index 000000000..22b3e0794--- /dev/null+++ b/model/models/gptoss/model.go@@ -0,0 +1,268 @@+package gptoss++import (+ "cmp"+ "math"+ "strings"++ "github.com/ollama/ollama/fs"+ "github.com/ollama/ollama/kvcache"+ "github.com/ollama/ollama/ml"+ "github.com/ollama/ollama/ml/nn"+ "github.com/ollama/ollama/ml/nn/fast"+ "github.com/ollama/ollama/ml/nn/rope"+ "github.com/ollama/ollama/model"+ "github.com/ollama/ollama/model/input"+)++type Transformer struct {+ model.Base+ model.BytePairEncoding++ TokenEmbedding *nn.Embedding `gguf:"token_embd"`+ TransformerBlocks []TransformerBlock `gguf:"blk"`+ OutputNorm *nn.RMSNorm `gguf:"output_norm"`+ Output *nn.Linear `gguf:"output,alt:token_embd"`++ Options+}++// Forward implements model.Model.+func (m *Transformer) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {+ hiddenStates := m.TokenEmbedding.Forward(ctx, batch.Inputs)+ positions := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))++ one := ctx.Input().FromFloatSlice([]float32{1}, 1)+ for i, block := range m.TransformerBlocks {+ m.Cache.SetLayer(i)+ if c, ok := m.Cache.(*kvcache.WrapperCache); ok {+ // Even layers are sliding window attention.+ c.SetLayerType(i % 2)+ }++ var outputs ml.Tensor+ if len(batch.Outputs) > 0 && i == len(m.TransformerBlocks)-1 {+ outputs = ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs))+ }++ hiddenStates = block.Forward(ctx, hiddenStates, positions, outputs, one, m.Cache, &m.Options)+ }++ hiddenStates = m.OutputNorm.Forward(ctx, hiddenStates, m.eps)+ return m.Output.Forward(ctx, hiddenStates), nil+}++func (m *Transformer) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {+ return fast.RoPE(ctx, key, shift, m.headDim(), m.ropeBase, 1./m.ropeScale, m.RoPEOptions()...), nil+}++type Options struct {+ hiddenSize,+ numHeads,+ numKVHeads,+ keyLength,+ valueLength,+ numExperts,+ numExpertsUsed,+ originalContextLength int++ eps,+ ropeBase,+ ropeScale float32+}++func (o Options) RoPEOptions() []func(*rope.Options) {+ return []func(*rope.Options){+ rope.WithTypeNeoX(),+ rope.WithOriginalContextLength(o.originalContextLength),+ rope.WithExtrapolationFactor(1.),+ // NOTE: ggml sets this implicitly so there's no need to set it here+ // rope.WithAttentionFactor(0.1*float32(math.Log(float64(o.ropeScale))) + 1.0),+ }+}++func (o Options) headDim() int {+ return cmp.Or(o.keyLength, o.valueLength, o.hiddenSize/o.numHeads)+}++type TransformerBlock struct {+ Attention *AttentionBlock+ MLP *MLPBlock+}++func (d *TransformerBlock) Forward(ctx ml.Context, hiddenStates, positions, outputs, one ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor {+ hiddenStates = d.Attention.Forward(ctx, hiddenStates, positions, cache, opts)+ if outputs != nil {+ hiddenStates = hiddenStates.Rows(ctx, outputs)+ }++ hiddenStates = d.MLP.Forward(ctx, hiddenStates, one, opts)+ return hiddenStates+}++type AttentionBlock struct {+ Norm *nn.RMSNorm `gguf:"attn_norm"`+ QKV *nn.Linear `gguf:"attn_qkv"`+ Output *nn.Linear `gguf:"attn_out"`+ Sinks ml.Tensor `gguf:"attn_sinks"`+}++func (attn *AttentionBlock) Forward(ctx ml.Context, hiddenStates, positions ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor {+ batchSize := hiddenStates.Dim(1)++ residual := hiddenStates+ hiddenStates = attn.Norm.Forward(ctx, hiddenStates, opts.eps)++ qkv := attn.QKV.Forward(ctx, hiddenStates)++ // query = qkv[..., : num_attention_heads * head_dim].reshape(batch_size, num_attention_heads, head_dim)+ query := qkv.View(ctx,+ 0,+ opts.headDim(), qkv.Stride(0)*opts.headDim(),+ opts.numHeads, qkv.Stride(1),+ batchSize,+ )+ query = fast.RoPE(ctx, query, positions, opts.headDim(), opts.ropeBase, 1./opts.ropeScale, opts.RoPEOptions()...)++ // key = qkv[..., num_attention_heads * head_dim:(num_attention_heads + num_key_value_heads) * head_dim].reshape(batch_size, num_key_value_heads, head_dim)+ key := qkv.View(ctx,+ qkv.Stride(0)*opts.headDim()*opts.numHeads,+ opts.headDim(), qkv.Stride(0)*opts.headDim(),+ opts.numKVHeads, qkv.Stride(1),+ batchSize,+ )+ key = fast.RoPE(ctx, key, positions, opts.headDim(), opts.ropeBase, 1./opts.ropeScale, opts.RoPEOptions()...)++ // value = qkv[..., (num_attention_heads + num_key_value_heads) * head_dim:].reshape(batch_size, num_key_value_heads, head_dim)+ value := qkv.View(ctx,+ qkv.Stride(0)*opts.headDim()*(opts.numHeads+opts.numKVHeads),+ opts.headDim(), qkv.Stride(0)*opts.headDim(),+ opts.numKVHeads, qkv.Stride(1),+ batchSize,+ )++ cache.Put(ctx, key, value)+ key, value, mask := cache.Get(ctx)++ query = query.Permute(ctx, 0, 2, 1, 3)+ key = key.Permute(ctx, 0, 2, 1, 3)++ scores := key.MulmatFullPrec(ctx, query)+ scores = scores.Scale(ctx, 1./math.Sqrt(float64(opts.headDim())))+ scores = scores.Add(ctx, mask)++ scores = scores.Concat(ctx, attn.Sinks.Reshape(ctx, 1, 1, opts.numHeads, 1).Repeat(ctx, 1, batchSize), 0)+ scores = scores.Softmax(ctx)+ scores = scores.Pad(ctx, -1, 0, 0, 0)++ attention := value.Mulmat(ctx, scores)+ attention = attention.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)+ attention = attention.Reshape(ctx, attention.Dim(0)*attention.Dim(1), batchSize)++ return attn.Output.Forward(ctx, attention).Add(ctx, residual)+}++type MLPBlock struct {+ Norm *nn.RMSNorm `gguf:"ffn_norm"`+ Router *nn.Linear `gguf:"ffn_gate_inp"`+ GateUp *nn.LinearBatch `gguf:"ffn_gate_up_exps"`+ Down *nn.LinearBatch `gguf:"ffn_down_exps"`+}++func (mlp *MLPBlock) Forward(ctx ml.Context, hiddenStates, one ml.Tensor, opts *Options) ml.Tensor {+ hiddenDim, sequenceLength, batchSize := hiddenStates.Dim(0), hiddenStates.Dim(1), hiddenStates.Dim(2)++ residual := hiddenStates+ hiddenStates = mlp.Norm.Forward(ctx, hiddenStates, opts.eps)++ hiddenStates = hiddenStates.Reshape(ctx, hiddenDim, sequenceLength*batchSize)+ routingWeights := mlp.Router.Forward(ctx, hiddenStates)++ selectedExperts := routingWeights.TopK(ctx, opts.numExpertsUsed)+ routingWeights = routingWeights.Reshape(ctx, 1, opts.numExperts, sequenceLength*batchSize).Rows(ctx, selectedExperts)+ routingWeights = routingWeights.Reshape(ctx, opts.numExpertsUsed, sequenceLength*batchSize).Softmax(ctx)+ routingWeights = routingWeights.Reshape(ctx, 1, opts.numExpertsUsed, sequenceLength*batchSize)++ hiddenStates = hiddenStates.Reshape(ctx, hiddenStates.Dim(0), 1, hiddenStates.Dim(1))++ hiddenStates = mlp.GateUp.Forward(ctx, hiddenStates, selectedExperts)+ hiddenStates = hiddenStates.Reshape(ctx, 2, hiddenStates.Dim(0)/2, hiddenStates.Dim(1), hiddenStates.Dim(2))++ dimStride := []int{hiddenStates.Dim(0) / 2, hiddenStates.Stride(1), hiddenStates.Dim(1), hiddenStates.Stride(2), hiddenStates.Dim(2), hiddenStates.Stride(3), hiddenStates.Dim(3)}++ glu := hiddenStates.View(ctx, 0, dimStride...)+ glu = glu.Contiguous(ctx)+ glu = glu.Clamp(ctx, float32(math.Inf(-1)), 7.0)+ glu = glu.QuickGELU(ctx)++ linear := hiddenStates.View(ctx, hiddenStates.Stride(0), dimStride...)+ linear = linear.Clamp(ctx, -7.0, 7.0)++ hiddenStates = glu.Mul(ctx, linear.Add(ctx, one))+ hiddenStates = hiddenStates.Reshape(ctx, hiddenStates.Dim(0)*hiddenStates.Dim(1), hiddenStates.Dim(2), hiddenStates.Dim(3))++ experts := mlp.Down.Forward(ctx, hiddenStates, selectedExperts)+ experts = experts.Mul(ctx, routingWeights)++ nextStates := experts.View(ctx, 0, experts.Dim(0), experts.Stride(2), experts.Dim(2))+ for i := 1; i < opts.numExpertsUsed; i++ {+ nextStates = nextStates.Add(ctx, experts.View(ctx, i*experts.Stride(1), experts.Dim(0), experts.Stride(2), experts.Dim(2)))+ }++ return nextStates.Add(ctx, residual)+}++func New(c fs.Config) (model.Model, error) {+ m := Transformer{+ TransformerBlocks: make([]TransformerBlock, c.Uint("block_count")),+ BytePairEncoding: model.NewBytePairEncoding(+ c.String("tokenizer.ggml.pretokenizer",+ strings.Join([]string{+ `[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?`,+ `[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?`,+ `\p{N}{1,3}`,+ ` ?[^\s\p{L}\p{N}]+[\r\n/]*`,+ `\s*[\r\n]+`,+ `\s+(?!\S)`,+ `\s+`,+ }, "|"),+ ),+ &model.Vocabulary{+ Values: c.Strings("tokenizer.ggml.tokens"),+ Types: c.Ints("tokenizer.ggml.token_type"),+ Merges: c.Strings("tokenizer.ggml.merges"),+ AddBOS: c.Bool("tokenizer.ggml.add_bos_token", false),+ BOS: []int32{int32(c.Uint("tokenizer.ggml.bos_token_id"))},+ AddEOS: c.Bool("tokenizer.ggml.add_eos_token", false),+ EOS: append(+ []int32{int32(c.Uint("tokenizer.ggml.eos_token_id"))},+ c.Ints("tokenizer.ggml.eos_token_ids")...,+ ),+ },+ ),+ Options: Options{+ hiddenSize: int(c.Uint("embedding_length")),+ numHeads: int(c.Uint("attention.head_count")),+ numKVHeads: int(c.Uint("attention.head_count_kv")),+ keyLength: int(c.Uint("attention.key_length")),+ valueLength: int(c.Uint("attention.value_length")),+ numExperts: int(c.Uint("expert_count")),+ numExpertsUsed: int(c.Uint("expert_used_count")),+ eps: c.Float("attention.layer_norm_rms_epsilon"),+ ropeBase: c.Float("rope.freq_base"),+ ropeScale: c.Float("rope.scaling.factor", 1.),+ originalContextLength: int(c.Uint("rope.scaling.original_context_length")),+ },+ }++ m.Cache = kvcache.NewWrapperCache(+ kvcache.NewSWAMemCache(int32(c.Uint("attention.sliding_window")), 4096, m.Shift),+ kvcache.NewCausalCache(m.Shift),+ )+ m.Cache.SetConfig(ml.CacheConfig{CachePadding: 32, PermutedV: true})+ return &m, nil+}++func init() {+ model.Register("gptoss", New)+}diff --git a/model/models/models.go b/model/models/models.goindex 8752878e2..c880a4720 100644--- a/model/models/models.go+++ b/model/models/models.go@@ -4,6 +4,7 @@ import (_ "github.com/ollama/ollama/model/models/gemma2"_ "github.com/ollama/ollama/model/models/gemma3"_ "github.com/ollama/ollama/model/models/gemma3n"+ _ "github.com/ollama/ollama/model/models/gptoss"_ "github.com/ollama/ollama/model/models/llama"_ "github.com/ollama/ollama/model/models/llama4"_ "github.com/ollama/ollama/model/models/mistral3"diff --git a/openai/openai.go b/openai/openai.goindex 35b8b9a01..d065de8f1 100644--- a/openai/openai.go+++ b/openai/openai.go@@ -36,6 +36,7 @@ type ErrorResponse struct {type Message struct {Role string `json:"role"`Content any `json:"content"`+ Reasoning string `json:"reasoning,omitempty"`ToolCalls []ToolCall `json:"tool_calls,omitempty"`}@@ -81,6 +82,10 @@ type StreamOptions struct {IncludeUsage bool `json:"include_usage"`}+type Reasoning struct {+ Effort *string `json:"effort,omitempty"`+}+type ChatCompletionRequest struct {Model string `json:"model"`Messages []Message `json:"messages"`@@ -95,6 +100,7 @@ type ChatCompletionRequest struct {TopP *float64 `json:"top_p"`ResponseFormat *ResponseFormat `json:"response_format"`Tools []api.Tool `json:"tools"`+ Reasoning *Reasoning `json:"reasoning,omitempty"`}type ChatCompletion struct {@@ -253,7 +259,7 @@ func toChatCompletion(id string, r api.ChatResponse) ChatCompletion {SystemFingerprint: "fp_ollama",Choices: []Choice{{Index: 0,- Message: Message{Role: r.Message.Role, Content: r.Message.Content, ToolCalls: toolCalls},+ Message: Message{Role: r.Message.Role, Content: r.Message.Content, ToolCalls: toolCalls, Reasoning: r.Message.Thinking},FinishReason: func(reason string) *string {if len(toolCalls) > 0 {reason = "tool_calls"@@ -278,10 +284,10 @@ func toChunk(id string, r api.ChatResponse, toolCallSent bool) ChatCompletionChuSystemFingerprint: "fp_ollama",Choices: []ChunkChoice{{Index: 0,- Delta: Message{Role: "assistant", Content: r.Message.Content, ToolCalls: toolCalls},+ Delta: Message{Role: "assistant", Content: r.Message.Content, ToolCalls: toolCalls, Reasoning: r.Message.Thinking},FinishReason: func(reason string) *string {if len(reason) > 0 {- if toolCallSent {+ if toolCallSent || len(toolCalls) > 0 {return &finishReasonToolCalls}return &reason@@ -397,7 +403,7 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {for _, msg := range r.Messages {switch content := msg.Content.(type) {case string:- messages = append(messages, api.Message{Role: msg.Role, Content: content})+ messages = append(messages, api.Message{Role: msg.Role, Content: content, Thinking: msg.Reasoning})case []any:for _, c := range content {data, ok := c.(map[string]any)@@ -508,6 +514,10 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {options["top_p"] = 1.0}+ if r.Reasoning != nil {+ options["reasoning"] = *r.Reasoning.Effort+ }+var format json.RawMessageif r.ResponseFormat != nil {switch strings.ToLower(strings.TrimSpace(r.ResponseFormat.Type)) {@@ -521,6 +531,13 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {}}+ var think *api.ThinkValue+ if r.Reasoning != nil {+ think = &api.ThinkValue{+ Value: *r.Reasoning.Effort,+ }+ }+return &api.ChatRequest{Model: r.Model,Messages: messages,@@ -528,6 +545,7 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {Options: options,Stream: &r.Stream,Tools: r.Tools,+ Think: think,}, nil}diff --git a/server/harmonyparser.go b/server/harmonyparser.gonew file mode 100644index 000000000..fd6c64e73--- /dev/null+++ b/server/harmonyparser.go@@ -0,0 +1,379 @@+package server++import (+ "context"+ "log/slog"+ "strings"+ "unicode"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/logutil"+)++type harmonyParserState int++const (+ harmonyParserState_LookingForMessageStart harmonyParserState = iota+ harmonyParserState_ParsingHeader+ harmonyParserState_ParsingContent+)++func shouldUseHarmony(model Model) bool {+ if model.Config.ModelFamily == "gptoss" {+ // heuristic to check whether the template expects to be parsed via harmony:+ // search for harmony tags that are nearly always used+ if model.Template.Contains("<|start|>") && model.Template.Contains("<|end|>") {+ return true+ }+ }++ return false+}++func (s harmonyParserState) String() string {+ switch s {+ // we're looking for the message start tag+ case harmonyParserState_LookingForMessageStart:+ return "LookingForMessageStart"+ case harmonyParserState_ParsingHeader:+ return "ParsingHeader"+ case harmonyParserState_ParsingContent:+ return "ParsingContent"+ default:+ return "Unknown"+ }+}++type HarmonyParser struct {+ state harmonyParserState+ MessageStartTag string+ MessageEndTag string+ HeaderEndTag string+ acc strings.Builder+ lifetimeAcc strings.Builder+}++type HarmonyEvent interface {+ isHarmonyEvent()+}++type HarmonyEventMessageStart struct{}++func (HarmonyEventMessageStart) isHarmonyEvent() {}++type HarmonyEventHeaderComplete struct {+ Header HarmonyHeader+}++func (HarmonyEventHeaderComplete) isHarmonyEvent() {}++type HarmonyEventContentEmitted struct {+ Content string+}++func (HarmonyEventContentEmitted) isHarmonyEvent() {}++type HarmonyEventMessageEnd struct{}++func (HarmonyEventMessageEnd) isHarmonyEvent() {}++type HarmonyHeader struct {+ Role string+ Channel string+ Recipient string+}++func (s *HarmonyParser) AddImplicitStart() {+ s.acc.WriteString("<|start|>assistant")+}++func (s *HarmonyParser) AddImplicitStartOrPrefill(lastMessage *api.Message) {+ if lastMessage != nil && lastMessage.Role == "assistant" {+ // handle prefilling conditions+ if lastMessage.Content != "" {+ s.acc.WriteString("<|start|>assistant<|channel|>final<|message|>")+ return+ } else if lastMessage.Thinking != "" {+ s.acc.WriteString("<|start|>assistant<|channel|>analysis<|message|>")+ return+ }+ }+ s.AddImplicitStart()+}++func (s *HarmonyParser) AddContent(content string) []HarmonyEvent {+ s.lifetimeAcc.WriteString(content)+ s.acc.WriteString(content)++ var events []HarmonyEvent++ keepLooping := true+ // we loop because we might pass through multiple parsing states in a single+ // call to addContent, and we want to make sure callers don't have to wait for+ // data that's already unambiguous+ for keepLooping {+ var newEvents []HarmonyEvent+ newEvents, keepLooping = eat(s)+ events = append(events, newEvents...)+ }++ return events+}++// the additional bool return is true iff we should continue eating+func eat(s *HarmonyParser) ([]HarmonyEvent, bool) {+ switch s.state {+ case harmonyParserState_LookingForMessageStart:+ // does the acc contain the message start tag?+ if strings.Contains(s.acc.String(), s.MessageStartTag) {+ // split the acc into the message start tag and the rest+ split := strings.SplitN(s.acc.String(), s.MessageStartTag, 2)+ before := split[0]+ if before != "" {+ slog.Warn("harmony parser: found message start tag in the middle of the content", "content", s.acc.String())+ }+ after := split[1]+ s.acc.Reset()+ s.acc.WriteString(after)+ s.state = harmonyParserState_ParsingHeader+ return []HarmonyEvent{HarmonyEventMessageStart{}}, true+ }++ // no match, so we keep accumulating+ return nil, false+ case harmonyParserState_ParsingHeader:+ if strings.Contains(s.acc.String(), s.HeaderEndTag) {+ split := strings.SplitN(s.acc.String(), s.HeaderEndTag, 2)+ header := split[0]+ after := split[1]+ s.acc.Reset()+ s.acc.WriteString(after)+ s.state = harmonyParserState_ParsingContent+ return []HarmonyEvent{HarmonyEventHeaderComplete{Header: s.parseHeader(header)}}, true+ }+ return nil, false+ case harmonyParserState_ParsingContent:+ if strings.Contains(s.acc.String(), s.MessageEndTag) {+ // if we already have the message end tag, we can emit the content up to it+ split := strings.SplitN(s.acc.String(), s.MessageEndTag, 2)+ content := split[0]+ after := split[1]+ s.acc.Reset()+ s.acc.WriteString(after)+ s.state = harmonyParserState_LookingForMessageStart+ events := []HarmonyEvent{}+ if content != "" {+ events = append(events, HarmonyEventContentEmitted{Content: content})+ }+ events = append(events, HarmonyEventMessageEnd{})+ return events, true+ } else if overlapLen := overlap(s.acc.String(), s.MessageEndTag); overlapLen > 0 {+ // if our suffix contains the start of the message end tag, we can emit+ // the content up to the start of the message end tag+ content := s.acc.String()[:len(s.acc.String())-overlapLen]+ remaining := s.acc.String()[len(s.acc.String())-overlapLen:]+ s.acc.Reset()+ s.acc.WriteString(remaining)+ // emit the content we know isn't part of the message end tag, and keep+ // accumulating to disambiguate the rest+ if content == "" {+ return nil, false+ }+ return []HarmonyEvent{HarmonyEventContentEmitted{Content: content}}, false+ } else {+ // no end tag, so it's still normal content that we can immediately emit+ content := s.acc.String()+ if content == "" {+ return nil, false+ }+ s.acc.Reset()+ return []HarmonyEvent{HarmonyEventContentEmitted{Content: content}}, false+ }+ }++ return nil, false+}++func (s *HarmonyParser) parseHeader(raw string) HarmonyHeader {+ harmonyHeader := HarmonyHeader{}++ // if `<|constrain|>` is present, ensure it has a space before it so it gets+ // parsed as a separate token, even if the model didn't include the space+ if strings.Contains(raw, "<|constrain|>") {+ raw = strings.Replace(raw, "<|constrain|>", " <|constrain|>", 1)+ raw = strings.TrimSpace(raw)+ }++ // look for the optional channel tag, which is `<|channel|>` followed by the+ // channel name, all without any whitespace+ channelIndex := strings.Index(raw, "<|channel|>")+ if channelIndex != -1 {+ before := raw[:channelIndex]+ after := raw[channelIndex+len("<|channel|>"):]+ // the channel name is `after` all the way up to the first (if any) whitespace character+ idx := strings.IndexFunc(after, func(r rune) bool {+ return unicode.IsSpace(r)+ })+ if idx == -1 {+ idx = len(after)+ }+ harmonyHeader.Channel = after[:idx]+ after = after[idx:]+ // now we remove the channel tag from the raw string to further process+ raw = before + after+ raw = strings.TrimSpace(raw)+ }++ // split the header into whitespace-separated tokens+ tokens := strings.Fields(raw)++ // the first token is treated as the role+ if len(tokens) == 0 {+ slog.Error("harmony parser: missing role in header", "header", raw)+ return harmonyHeader+ }+ role := tokens[0]+ tokens = tokens[1:]+ // special case: if role starts with to= then it's a tool call+ if strings.HasPrefix(role, "to=") {+ harmonyHeader.Recipient = role[3:]+ harmonyHeader.Role = "tool"+ } else {+ harmonyHeader.Role = role+ }++ // the recipient (if any) can be specified before or after the channel tag, so+ // we check it at the end once we've already parsed the channel and role+ if harmonyHeader.Recipient == "" && len(tokens) > 0 && strings.HasPrefix(tokens[0], "to=") {+ harmonyHeader.Recipient = tokens[0][3:]+ }++ return harmonyHeader+}++// longest overlap between suffix of s and prefix of delim+func overlap(s, delim string) int {+ max := min(len(delim), len(s))+ for i := max; i > 0; i-- {+ if strings.HasSuffix(s, delim[:i]) {+ return i+ }+ }+ return 0+}++// harmonyMessageState represents the current state of message processing+type harmonyMessageState int++const (+ harmonyMessageState_Normal harmonyMessageState = iota+ harmonyMessageState_Thinking+ harmonyMessageState_ToolCalling+)++// HarmonyMessageHandler processes harmony events and accumulates content appropriately.+// This is a higher level interface that maps harmony concepts into ollama concepts+type HarmonyMessageHandler struct {+ state harmonyMessageState+ harmonyParser *HarmonyParser+}++// NewHarmonyMessageHandler creates a new message handler+func NewHarmonyMessageHandler() *HarmonyMessageHandler {+ return &HarmonyMessageHandler{+ state: harmonyMessageState_Normal,+ harmonyParser: &HarmonyParser{+ MessageStartTag: "<|start|>",+ MessageEndTag: "<|end|>",+ HeaderEndTag: "<|message|>",+ },+ }+}++// AddContent processes the content and returns the content, thinking, and tool content.+// content and thinking are already fully parsed, but tool content still needs to be passed to the tool parser+func (h *HarmonyMessageHandler) AddContent(content string, toolParser *HarmonyToolCallAccumulator) (string, string, string) {+ contentSb := strings.Builder{}+ thinkingSb := strings.Builder{}+ toolContentSb := strings.Builder{}++ events := h.harmonyParser.AddContent(content)+ for _, event := range events {+ switch event := event.(type) {+ case HarmonyEventHeaderComplete:+ slog.Log(context.TODO(), logutil.LevelTrace, "harmony event header complete", "header", event.Header)+ switch event.Header.Channel {+ case "analysis":+ if event.Header.Recipient != "" {+ h.state = harmonyMessageState_ToolCalling+ // event.Header.Recipient is the tool name, something like+ // "browser.search" for a built-in, or "functions.calc" for a+ // custom one+ toolParser.SetToolName(event.Header.Recipient)+ } else {+ h.state = harmonyMessageState_Thinking+ }+ case "commentary":+ if event.Header.Recipient != "" {+ h.state = harmonyMessageState_ToolCalling+ toolParser.SetToolName(event.Header.Recipient)+ } else {+ h.state = harmonyMessageState_Normal+ }+ case "final":+ h.state = harmonyMessageState_Normal+ }+ case HarmonyEventContentEmitted:+ slog.Log(context.TODO(), logutil.LevelTrace, "harmony event content", "content", event.Content, "state", h.state)+ if h.state == harmonyMessageState_Normal {+ contentSb.WriteString(event.Content)+ } else if h.state == harmonyMessageState_Thinking {+ thinkingSb.WriteString(event.Content)+ } else if h.state == harmonyMessageState_ToolCalling {+ toolContentSb.WriteString(event.Content)+ }+ case HarmonyEventMessageEnd:+ h.state = harmonyMessageState_Normal+ }+ }+ return contentSb.String(), thinkingSb.String(), toolContentSb.String()+}++func (h *HarmonyMessageHandler) CreateToolParser() *HarmonyToolCallAccumulator {+ return &HarmonyToolCallAccumulator{+ state: harmonyToolCallState_Normal,+ currentToolName: nil,+ }+}++type harmonyToolCallState int++const (+ harmonyToolCallState_Normal harmonyToolCallState = iota+ harmonyToolCallState_ToolCalling+)++type HarmonyToolCallAccumulator struct {+ state harmonyToolCallState+ acc strings.Builder+ currentToolName *string+}++func (a *HarmonyToolCallAccumulator) SetToolName(toolName string) {+ a.currentToolName = &toolName+}++func (a *HarmonyToolCallAccumulator) Add(content string) {+ a.acc.WriteString(content)+}++func (a *HarmonyToolCallAccumulator) Drain() (*string, string) {+ str := a.acc.String()+ a.state = harmonyToolCallState_Normal+ a.acc.Reset()+ return a.currentToolName, str+}++func (a *HarmonyToolCallAccumulator) Content() string {+ return a.acc.String()+}diff --git a/server/harmonyparser_test.go b/server/harmonyparser_test.gonew file mode 100644index 000000000..cd1743e1c--- /dev/null+++ b/server/harmonyparser_test.go@@ -0,0 +1,469 @@+package server++import (+ "fmt"+ "reflect"+ "testing"+)++func TestHeaderParsing(t *testing.T) {+ tests := []struct {+ in, wantRole, wantChannel, wantRecipient string+ }{+ {+ in: "assistant<|channel|>analysis",+ wantRole: "assistant",+ wantChannel: "analysis",+ wantRecipient: "",+ },+ {+ in: "assistant<|channel|>analysis to=functions.get_weather",+ wantRole: "assistant",+ wantChannel: "analysis",+ wantRecipient: "functions.get_weather",+ },+ {+ in: "assistant to=functions.get_weather<|channel|>analysis",+ wantRole: "assistant",+ wantChannel: "analysis",+ wantRecipient: "functions.get_weather",+ },+ // special case where the role is replaced by the recipient (matches reference code)+ {+ in: "to=functions.get_weather<|channel|>analysis",+ wantRole: "tool",+ wantChannel: "analysis",+ wantRecipient: "functions.get_weather",+ },+ // extra token after the recipient is ignored+ {+ in: "assistant to=functions.get_weather abc<|channel|>analysis",+ wantRole: "assistant",+ wantChannel: "analysis",+ wantRecipient: "functions.get_weather",+ },+ // with constrain tag, recipient after channel tag+ {+ in: "assistant<|channel|>commentary to=functions.get_weather <|constrain|>json",+ wantRole: "assistant",+ wantChannel: "commentary",+ wantRecipient: "functions.get_weather",+ },+ // with constrain tag, recipient before channel tag+ {+ in: "assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",+ wantRole: "assistant",+ wantChannel: "commentary",+ wantRecipient: "functions.get_weather",+ },+ // constrain tag without space+ {+ in: "assistant<|channel|>commentary to=functions.get_weather<|constrain|>json",+ wantRole: "assistant",+ wantChannel: "commentary",+ wantRecipient: "functions.get_weather",+ },+ // constrain tag without space, different order+ {+ in: "assistant to=functions.get_weather<|channel|>commentary<|constrain|>json",+ wantRole: "assistant",+ wantChannel: "commentary",+ wantRecipient: "functions.get_weather",+ },+ }+ for i, tt := range tests {+ parser := HarmonyParser{+ MessageStartTag: "<|start|>",+ MessageEndTag: "<|end|>",+ HeaderEndTag: "<|message|>",+ }+ header := parser.parseHeader(tt.in)++ if header.Role != tt.wantRole {+ t.Errorf("case %d: got role \"%s\", want \"%s\"", i, header.Role, tt.wantRole)+ }+ if header.Channel != tt.wantChannel {+ t.Errorf("case %d: got channel \"%s\", want \"%s\"", i, header.Channel, tt.wantChannel)+ }+ if header.Recipient != tt.wantRecipient {+ t.Errorf("case %d: got recipient \"%s\", want \"%s\"", i, header.Recipient, tt.wantRecipient)+ }+ }+}++func TestHarmonyParserHeaderEvent(t *testing.T) {+ tests := []struct {+ in, wantRole, wantChannel, wantRecipient string+ implicitStart bool+ }{+ {+ in: "<|start|>user<|message|>What is 2 + 2?<|end|>",+ wantRole: "user",+ wantChannel: "",+ wantRecipient: "",+ },+ {+ in: "<|start|>assistant<|channel|>analysis<|message|>What is 2 + 2?<|end|>",+ wantRole: "assistant",+ wantChannel: "analysis",+ wantRecipient: "",+ },+ {+ in: "<|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{\"location\":\"San Francisco\"}<|call|><|start|>functions.get_weather to=assistant<|message|>{\"sunny\": true, \"temperature\": 20}<|end|>",+ wantRole: "assistant",+ wantChannel: "commentary",+ wantRecipient: "functions.get_weather",+ },+ {+ in: "<|channel|>analysis<|message|>User asks weather in SF. We need location. Use get_current_weather with location \"San Francisco, CA\".<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\"location\":\"San Francisco, CA\"}<|call|>",+ wantRole: "assistant",+ wantChannel: "analysis",+ wantRecipient: "",+ implicitStart: true,+ },+ }+ for i, tt := range tests {+ parser := HarmonyParser{+ MessageStartTag: "<|start|>",+ MessageEndTag: "<|end|>",+ HeaderEndTag: "<|message|>",+ }+ if tt.implicitStart {+ parser.AddImplicitStart()+ }+ gotEvents := parser.AddContent(tt.in)+ if len(gotEvents) == 0 {+ t.Errorf("case %d: got no events, want at least one", i)+ }++ var firstHeaderEvent *HarmonyEventHeaderComplete+ // print events+ for _, event := range gotEvents {+ fmt.Printf("event: %+v\n", event)+ }+ for _, event := range gotEvents {+ if event, ok := event.(HarmonyEventHeaderComplete); ok {+ firstHeaderEvent = &event+ break+ }+ }++ if firstHeaderEvent == nil {+ t.Errorf("case %d: got no header complete event, want one", i)+ continue+ }+ gotHeader := firstHeaderEvent.Header+ if gotHeader.Role != tt.wantRole || gotHeader.Channel != tt.wantChannel || gotHeader.Recipient != tt.wantRecipient {+ t.Errorf("case %d: got header %+v, want role=%s channel=%s recipient=%s", i, gotHeader, tt.wantRole, tt.wantChannel, tt.wantRecipient)+ }+ }+}++func TestHarmonyParserNonStreaming(t *testing.T) {+ tests := []struct {+ in string+ implicitStart bool+ wantEvents []HarmonyEvent+ }{+ {+ in: "<|start|>user<|message|>What is 2 + 2?<|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "What is 2 + 2?"},+ HarmonyEventMessageEnd{},+ },+ },+ {+ in: "<|start|>assistant<|channel|>analysis<|message|>The answer is 4<|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "analysis", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "The answer is 4"},+ HarmonyEventMessageEnd{},+ },+ },+ {+ in: "<|start|>assistant<|channel|>commentary to=functions.calc<|message|>Computing...<|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "commentary", Recipient: "functions.calc"}},+ HarmonyEventContentEmitted{Content: "Computing..."},+ HarmonyEventMessageEnd{},+ },+ },+ {+ in: "<|start|>user<|message|><|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}},+ HarmonyEventMessageEnd{},+ },+ },+ {+ in: "<|start|>user<|message|>Hello<|end|><|start|>assistant<|message|>Hi!<|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "Hello"},+ HarmonyEventMessageEnd{},+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "Hi!"},+ HarmonyEventMessageEnd{},+ },+ },+ {+ in: "<|channel|>analysis<|message|>Thinking about the request<|end|>",+ implicitStart: true,+ wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}, HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "analysis", Recipient: ""}}, HarmonyEventContentEmitted{Content: "Thinking about the request"}, HarmonyEventMessageEnd{}},+ },+ }+ for i, tt := range tests {+ parser := HarmonyParser{+ MessageStartTag: "<|start|>",+ MessageEndTag: "<|end|>",+ HeaderEndTag: "<|message|>",+ }+ if tt.implicitStart {+ parser.AddImplicitStart()+ }+ gotEvents := parser.AddContent(tt.in)+ if !reflect.DeepEqual(gotEvents, tt.wantEvents) {+ t.Errorf("case %d: got events %#v, want %#v", i, gotEvents, tt.wantEvents)+ }+ }+}++func TestHarmonyParserStreaming(t *testing.T) {+ type step struct {+ input string+ wantEvents []HarmonyEvent+ }++ cases := []struct {+ desc string+ implicitStart bool+ steps []step+ }{+ {+ desc: "simple message streamed character by character",+ steps: []step{+ {+ input: "<",+ wantEvents: nil,+ },+ {+ input: "|",+ wantEvents: nil,+ },+ {+ input: "start|>u",+ wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}},+ },+ {+ input: "ser<|mess",+ wantEvents: nil,+ },+ {+ input: "age|>Hi",+ wantEvents: []HarmonyEvent{+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "Hi"},+ },+ },+ {+ input: " there",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: " there"}},+ },+ {+ input: "<|e",+ wantEvents: nil,+ },+ {+ input: "nd|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ {+ desc: "message with channel streamed",+ steps: []step{+ {+ input: "<|start|>assistant",+ wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}},+ },+ {+ input: "<|chan",+ wantEvents: nil,+ },+ {+ input: "nel|>analysis",+ wantEvents: nil,+ },+ {+ input: "<|message|>",+ wantEvents: []HarmonyEvent{HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "analysis", Recipient: ""}}},+ },+ {+ input: "Thinking",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "Thinking"}},+ },+ {+ input: "...",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "..."}},+ },+ {+ input: "<|end|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ {+ desc: "message with channel and recipient",+ steps: []step{+ {+ input: "<|start|>assistant<|channel|>commentary to=functions.calc<|message|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "commentary", Recipient: "functions.calc"}},+ },+ },+ {+ input: "{\"x\": 5}",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "{\"x\": 5}"}},+ },+ {+ input: "<|end|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ {+ desc: "message with channel and recipient (receipient before channel)",+ steps: []step{+ {+ input: "<|start|>assistant to=functions.calc<|channel|>commentary<|message|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "commentary", Recipient: "functions.calc"}},+ },+ },+ {+ input: "{\"x\": 5}",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "{\"x\": 5}"}},+ },+ {+ input: "<|end|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ {+ desc: "implicit start with channel",+ implicitStart: true,+ steps: []step{+ {+ input: "<|channel|>thinking",+ wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}},+ },+ {+ input: "<|message|>",+ wantEvents: []HarmonyEvent{HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "thinking", Recipient: ""}}},+ },+ {+ input: "Processing request",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "Processing request"}},+ },+ {+ input: "<|end|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ {+ desc: "multiple messages streamed",+ steps: []step{+ {+ input: "<|start|>user<|message|>Hello<|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "Hello"},+ HarmonyEventMessageEnd{},+ },+ },+ {+ input: "<|start|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}},+ },+ {+ input: "assistant<|message|>",+ wantEvents: []HarmonyEvent{HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "", Recipient: ""}}},+ },+ {+ input: "Hi!",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "Hi!"}},+ },+ {+ input: "<|end|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ {+ desc: "empty message",+ steps: []step{+ {+ input: "<|start|>system<|message|><|end|>",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "system", Channel: "", Recipient: ""}},+ HarmonyEventMessageEnd{},+ },+ },+ },+ },+ {+ desc: "partial tag that looks like end but isn't",+ steps: []step{+ {+ input: "<|start|>user<|message|>test<|e",+ wantEvents: []HarmonyEvent{+ HarmonyEventMessageStart{},+ HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}},+ HarmonyEventContentEmitted{Content: "test"},+ },+ },+ {+ input: "xample|>more",+ wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "<|example|>more"}},+ },+ {+ input: "<|end|>",+ wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}},+ },+ },+ },+ }++ for _, tc := range cases {+ t.Run(tc.desc, func(t *testing.T) {+ parser := HarmonyParser{+ MessageStartTag: "<|start|>",+ MessageEndTag: "<|end|>",+ HeaderEndTag: "<|message|>",+ }+ if tc.implicitStart {+ parser.AddImplicitStart()+ }++ for i, step := range tc.steps {+ gotEvents := parser.AddContent(step.input)+ if !reflect.DeepEqual(gotEvents, step.wantEvents) {+ t.Errorf("step %d: input %q: got events %#v, want %#v", i, step.input, gotEvents, step.wantEvents)+ }+ }+ })+ }+}diff --git a/server/images.go b/server/images.goindex 38505cc51..0c16dd435 100644--- a/server/images.go+++ b/server/images.go@@ -111,7 +111,8 @@ func (m *Model) Capabilities() []model.Capability {// Check for thinking capabilityopeningTag, closingTag := thinking.InferTags(m.Template.Template)- if openingTag != "" && closingTag != "" {+ hasTags := openingTag != "" && closingTag != ""+ if hasTags || m.Config.ModelFamily == "gptoss" {capabilities = append(capabilities, model.CapabilityThinking)}diff --git a/server/prompt.go b/server/prompt.goindex f8c895d71..5d6c3e27c 100644--- a/server/prompt.go+++ b/server/prompt.go@@ -19,7 +19,7 @@ type tokenizeFunc func(context.Context, string) ([]int, error)// chatPrompt accepts a list of messages and returns the prompt and images that should be used for the next chat turn.// chatPrompt truncates any messages that exceed the context window of the model, making sure to always include 1) the// latest message and 2) system messages-func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool, think *bool) (prompt string, images []llm.ImageData, _ error) {+func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool, think *api.ThinkValue) (prompt string, images []llm.ImageData, _ error) {var system []api.Message// TODO: Ideally we would compute this from the projector metadata but some pieces are implementation dependent@@ -42,11 +42,13 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.}thinkVal := false+ thinkLevel := ""if think != nil {- thinkVal = *think+ thinkVal = think.AsBool()+ thinkLevel = think.AsString()}var b bytes.Buffer- if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil {+ if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools, Think: thinkVal, ThinkLevel: thinkLevel, IsThinkSet: think != nil}); err != nil {return "", nil, err}@@ -101,10 +103,12 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.// truncate any messages that do not fit into the context windowvar b bytes.BufferthinkVal := false+ thinkLevel := ""if think != nil {- thinkVal = *think+ thinkVal = think.AsBool()+ thinkLevel = think.AsString()}- if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil {+ if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools, Think: thinkVal, ThinkLevel: thinkLevel, IsThinkSet: think != nil}); err != nil {return "", nil, err}diff --git a/server/prompt_test.go b/server/prompt_test.goindex 0043b9a47..659e64084 100644--- a/server/prompt_test.go+++ b/server/prompt_test.go@@ -209,7 +209,7 @@ func TestChatPrompt(t *testing.T) {model := tt.modelopts := api.Options{Runner: api.Runner{NumCtx: tt.limit}}think := false- prompt, images, err := chatPrompt(t.Context(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil, &think)+ prompt, images, err := chatPrompt(t.Context(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil, &api.ThinkValue{Value: think})if tt.error == nil && err != nil {t.Fatal(err)} else if tt.error != nil && err != tt.error {diff --git a/server/routes.go b/server/routes.goindex 40348e737..991e92003 100644--- a/server/routes.go+++ b/server/routes.go@@ -112,6 +112,11 @@ func (s *Server) scheduleRunner(ctx context.Context, name string, caps []model.Creturn nil, nil, nil, err}+ // This model requires a minimum context to function effectively+ if slices.Contains(model.Config.ModelFamilies, "gptoss") {+ opts.NumCtx = max(opts.NumCtx, 8192)+ }+runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)var runner *runnerRefselect {@@ -182,11 +187,26 @@ func (s *Server) GenerateHandler(c *gin.Context) {return}+ useHarmony := shouldUseHarmony(*m) && !req.Raw+ var harmonyMessageHandler *HarmonyMessageHandler+ var harmonyToolParser *HarmonyToolCallAccumulator+ if useHarmony {+ harmonyMessageHandler = NewHarmonyMessageHandler()+ harmonyMessageHandler.harmonyParser.AddImplicitStart()+ harmonyToolParser = harmonyMessageHandler.CreateToolParser()+ }++ // Validate Think value: string values currently only allowed for gptoss models+ if req.Think != nil && req.Think.IsString() && !useHarmony {+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("think value %q is not supported for this model", req.Think.AsString())})+ return+ }+caps := []model.Capability{model.CapabilityCompletion}if req.Suffix != "" {caps = append(caps, model.CapabilityInsert)}- if req.Think != nil && *req.Think {+ if req.Think != nil && req.Think.AsBool() {caps = append(caps, model.CapabilityThinking)// TODO(drifkin): consider adding a warning if it's false and the model// doesn't support thinking. It's not strictly required, but it can be a@@ -261,7 +281,11 @@ func (s *Server) GenerateHandler(c *gin.Context) {values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt})}- values.Think = req.Think != nil && *req.Think+ values.Think = req.Think != nil && req.Think.AsBool()+ values.ThinkLevel = ""+ if req.Think != nil {+ values.ThinkLevel = req.Think.AsString()+ }values.IsThinkSet = req.Think != nilvar b bytes.Buffer@@ -284,11 +308,13 @@ func (s *Server) GenerateHandler(c *gin.Context) {}var thinkingState *thinking.Parser- openingTag, closingTag := thinking.InferTags(m.Template.Template)- if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" {- thinkingState = &thinking.Parser{- OpeningTag: openingTag,- ClosingTag: closingTag,+ if !useHarmony {+ openingTag, closingTag := thinking.InferTags(m.Template.Template)+ if req.Think != nil && req.Think.AsBool() && openingTag != "" && closingTag != "" {+ thinkingState = &thinking.Parser{+ OpeningTag: openingTag,+ ClosingTag: closingTag,+ }}}@@ -316,7 +342,12 @@ func (s *Server) GenerateHandler(c *gin.Context) {},}- if thinkingState != nil {+ if useHarmony {+ content, thinking, toolContent := harmonyMessageHandler.AddContent(cr.Content, harmonyToolParser)+ res.Response = content+ res.Thinking = thinking+ harmonyToolParser.Add(toolContent)+ } else if thinkingState != nil {thinking, content := thinkingState.AddContent(cr.Content)res.Thinking = thinkingres.Response = content@@ -327,6 +358,25 @@ func (s *Server) GenerateHandler(c *gin.Context) {}if cr.Done {+ if useHarmony {+ toolName, toolContent := harmonyToolParser.Drain()+ if toolName != nil {+ *toolName = strings.TrimPrefix(*toolName, "functions.")+ var args api.ToolCallFunctionArguments+ if err := json.Unmarshal([]byte(toolContent), &args); err != nil {+ ch <- gin.H{"error parsing tool call": err.Error()}+ return+ }++ res.ToolCalls = append(res.ToolCalls, api.ToolCall{+ Function: api.ToolCallFunction{+ Name: *toolName,+ Arguments: args,+ },+ })+ }+ }+res.DoneReason = cr.DoneReason.String()res.TotalDuration = time.Since(checkpointStart)res.LoadDuration = checkpointLoaded.Sub(checkpointStart)@@ -341,6 +391,15 @@ func (s *Server) GenerateHandler(c *gin.Context) {}}+ if useHarmony {+ // only send messages with meaningful content (empty messages confuse clients)+ if res.Response != "" || res.Thinking != "" || res.Done || len(res.ToolCalls) > 0 {+ ch <- res+ }++ return+ }+ch <- res}); err != nil {ch <- gin.H{"error": err.Error()}@@ -1471,7 +1530,7 @@ func (s *Server) ChatHandler(c *gin.Context) {if len(req.Tools) > 0 {caps = append(caps, model.CapabilityTools)}- if req.Think != nil && *req.Think {+ if req.Think != nil && req.Think.AsBool() {caps = append(caps, model.CapabilityThinking)}@@ -1521,9 +1580,30 @@ func (s *Server) ChatHandler(c *gin.Context) {return}+ useHarmony := shouldUseHarmony(*m)++ // Validate Think value: string values currently only allowed for gptoss models+ if req.Think != nil && req.Think.IsString() && !useHarmony {+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("think value %q is not supported for this model", req.Think.AsString())})+ return+ }++ var harmonyMessageHandler *HarmonyMessageHandler+ var harmonyToolParser *HarmonyToolCallAccumulator++ if useHarmony {+ harmonyMessageHandler = NewHarmonyMessageHandler()+ var lastMessage *api.Message+ if len(msgs) > 0 {+ lastMessage = &msgs[len(msgs)-1]+ }+ harmonyMessageHandler.harmonyParser.AddImplicitStartOrPrefill(lastMessage)+ harmonyToolParser = harmonyMessageHandler.CreateToolParser()+ }+var thinkingState *thinking.ParseropeningTag, closingTag := thinking.InferTags(m.Template.Template)- if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" {+ if req.Think != nil && req.Think.AsBool() && openingTag != "" && closingTag != "" {thinkingState = &thinking.Parser{OpeningTag: openingTag,ClosingTag: closingTag,@@ -1531,7 +1611,7 @@ func (s *Server) ChatHandler(c *gin.Context) {}var toolParser *tools.Parser- if len(req.Tools) > 0 {+ if len(req.Tools) > 0 && !useHarmony {toolParser = tools.NewParser(m.Template.Template, req.Tools)}@@ -1557,6 +1637,38 @@ func (s *Server) ChatHandler(c *gin.Context) {EvalDuration: r.EvalDuration,},}+ if r.Done {+ res.DoneReason = r.DoneReason.String()+ res.TotalDuration = time.Since(checkpointStart)+ res.LoadDuration = checkpointLoaded.Sub(checkpointStart)+ }++ if useHarmony {+ content, thinking, toolContent := harmonyMessageHandler.AddContent(r.Content, harmonyToolParser)+ res.Message.Content = content+ res.Message.Thinking = thinking+ harmonyToolParser.Add(toolContent)++ if r.Done {+ toolName, toolContent := harmonyToolParser.Drain()+ if toolName != nil {+ *toolName = strings.TrimPrefix(*toolName, "functions.")+ var args api.ToolCallFunctionArguments+ if err := json.Unmarshal([]byte(toolContent), &args); err != nil {+ ch <- gin.H{"error parsing tool call": err.Error()}+ return+ }+ res.Message.ToolCalls = []api.ToolCall{{Function: api.ToolCallFunction{Name: *toolName, Arguments: args}}}+ }+ }++ // only send messages with meaningful content (empty messages confuse clients)+ if res.Message.Content != "" || res.Message.Thinking != "" || len(res.Message.ToolCalls) > 0 || res.Done {+ ch <- res+ }++ return+ }if thinkingState != nil {thinkingContent, remainingContent := thinkingState.AddContent(res.Message.Content)@@ -1568,12 +1680,6 @@ func (s *Server) ChatHandler(c *gin.Context) {res.Message.Thinking = thinkingContent}- if r.Done {- res.DoneReason = r.DoneReason.String()- res.TotalDuration = time.Since(checkpointStart)- res.LoadDuration = checkpointLoaded.Sub(checkpointStart)- }-if len(req.Tools) > 0 {toolCalls, content := toolParser.Add(res.Message.Content)if len(content) > 0 {diff --git a/server/routes_generate_test.go b/server/routes_generate_test.goindex 75a246fc6..477d6b814 100644--- a/server/routes_generate_test.go+++ b/server/routes_generate_test.go@@ -150,7 +150,7 @@ func TestGenerateChat(t *testing.T) {Messages: []api.Message{{Role: "user", Content: "Hello!"},},- Think: &think,+ Think: &api.ThinkValue{Value: think},})if w.Code != http.StatusBadRequest {diff --git a/server/routes_harmony_streaming_test.go b/server/routes_harmony_streaming_test.gonew file mode 100644index 000000000..503cb4d74--- /dev/null+++ b/server/routes_harmony_streaming_test.go@@ -0,0 +1,712 @@+package server++// this test file is to test integration of harmony parser into routes.go (as+// opposed to harmonyparser_test.go, which tests the parser in isolation)++import (+ "bytes"+ "context"+ "encoding/json"+ "net/http"+ "strings"+ "testing"+ "time"++ "github.com/gin-gonic/gin"+ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/discover"+ "github.com/ollama/ollama/fs/ggml"+ "github.com/ollama/ollama/llm"+)++func getTestTools() []api.Tool {+ return []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Description: "Get the current weather in a given location",+ Parameters: struct {+ Type string `json:"type"`+ Defs any `json:"$defs,omitempty"`+ Items any `json:"items,omitempty"`+ Required []string `json:"required"`+ Properties map[string]struct {+ Type api.PropertyType `json:"type"`+ Items any `json:"items,omitempty"`+ Description string `json:"description"`+ Enum []any `json:"enum,omitempty"`+ } `json:"properties"`+ }{+ Type: "object",+ Required: []string{"location"},+ Properties: map[string]struct {+ Type api.PropertyType `json:"type"`+ Items any `json:"items,omitempty"`+ Description string `json:"description"`+ Enum []any `json:"enum,omitempty"`+ }{+ "location": {+ Type: api.PropertyType{"string"},+ Description: "The city and state, e.g. San Francisco, CA",+ },+ },+ },+ },+ },+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "calculate",+ Description: "Calculate a mathematical expression",+ Parameters: struct {+ Type string `json:"type"`+ Defs any `json:"$defs,omitempty"`+ Items any `json:"items,omitempty"`+ Required []string `json:"required"`+ Properties map[string]struct {+ Type api.PropertyType `json:"type"`+ Items any `json:"items,omitempty"`+ Description string `json:"description"`+ Enum []any `json:"enum,omitempty"`+ } `json:"properties"`+ }{+ Type: "object",+ Required: []string{"expression"},+ Properties: map[string]struct {+ Type api.PropertyType `json:"type"`+ Items any `json:"items,omitempty"`+ Description string `json:"description"`+ Enum []any `json:"enum,omitempty"`+ }{+ "expression": {+ Type: api.PropertyType{"string"},+ Description: "The mathematical expression to calculate",+ },+ },+ },+ },+ },+ }+}++func createHarmonyTestModel(t *testing.T) (string, string) {+ t.Helper()++ return createBinFile(t, ggml.KV{+ "general.architecture": "gptoss",+ "llama.block_count": uint32(1),+ "llama.context_length": uint32(8192),+ "llama.embedding_length": uint32(4096),+ "llama.attention.head_count": uint32(32),+ "llama.attention.head_count_kv": uint32(8),+ "tokenizer.ggml.tokens": []string{""},+ "tokenizer.ggml.scores": []float32{0},+ "tokenizer.ggml.token_type": []int32{0},+ }, []*ggml.Tensor{+ {Name: "token_embd.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.attn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.ffn_down.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.ffn_gate.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.ffn_up.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.ffn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.attn_k.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.attn_output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.attn_q.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "blk.0.attn_v.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ {Name: "output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},+ })+}++// TestChatHarmonyParserStreamingRealtime verifies that chunks are emitted as soon as they're available+func TestChatHarmonyParserStreamingRealtime(t *testing.T) {+ gin.SetMode(gin.TestMode)++ type step struct {+ input llm.CompletionResponse+ wantContent string+ wantThinking string+ wantToolCalls []api.ToolCall+ }++ testCases := []struct {+ name string+ steps []step+ only bool+ }{+ {+ name: "content streams as it arrives",+ steps: []step{+ {+ input: llm.CompletionResponse{Content: "<|message|>Hello", Done: false},+ wantContent: "Hello",+ },+ {+ input: llm.CompletionResponse{Content: ", world", Done: false},+ wantContent: ", world",+ },+ {+ input: llm.CompletionResponse{Content: "!<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ wantContent: "!",+ },+ },+ },+ {+ name: "thinking streams separately from content",+ steps: []step{+ {+ input: llm.CompletionResponse{Content: "<|channel|>analysis<|message|>Thinking...", Done: false},+ wantThinking: "Thinking...",+ },+ {+ input: llm.CompletionResponse{Content: "<|end|>", Done: false},+ // No output expected - just closes the analysis message and resets state to normal+ },+ {+ input: llm.CompletionResponse{Content: "<|start|>assistant<|message|>Answer", Done: false},+ wantContent: "Answer", // After message end, state is reset to normal+ },+ {+ input: llm.CompletionResponse{Content: "<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ // No output expected - just closes the assistant message+ },+ },+ },+ {+ name: "partial tags buffer until complete",+ steps: []step{+ {+ input: llm.CompletionResponse{Content: "<|chan", Done: false},+ // No output - partial tag+ },+ {+ input: llm.CompletionResponse{Content: "nel|>analysis<|mess", Done: false},+ // No output - still building tags+ },+ {+ input: llm.CompletionResponse{Content: "age|>Deep ", Done: false},+ wantThinking: "Deep ",+ },+ {+ input: llm.CompletionResponse{Content: "thought<|end|>", Done: false},+ wantThinking: "thought",+ },+ {+ input: llm.CompletionResponse{Content: "<|start|>assistant<|message|>Done<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ wantContent: "Done", // After message end, state is reset to normal+ },+ },+ },+ {+ name: "simple assistant after analysis",+ steps: []step{+ {+ input: llm.CompletionResponse{Content: "<|channel|>analysis<|message|>Think<|end|><|start|>assistant<|message|>Answer<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ wantContent: "Answer",+ wantThinking: "Think",+ },+ },+ },+ {+ name: "tool call parsed and returned correctly",+ steps: []step{+ {+ input: llm.CompletionResponse{Content: "<|channel|>commentary to=functions.get_weather<|message|>{\"location\":\"San Francisco\"}<|end|><|start|>assistant<|message|>The weather is sunny<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ wantContent: "The weather is sunny",+ wantToolCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: api.ToolCallFunctionArguments{+ "location": "San Francisco",+ },+ },+ },+ },+ },+ },+ },+ {+ name: "tool call with streaming JSON across chunks",+ steps: []step{+ {+ input: llm.CompletionResponse{Content: "<|channel|>commentary to=functions.calculate<|message|>{\"expr", Done: false},+ // No output yet - incomplete JSON+ },+ {+ input: llm.CompletionResponse{Content: "ession\":\"2+", Done: false},+ // Still no output - incomplete JSON+ },+ {+ input: llm.CompletionResponse{Content: "2\"}", Done: true},+ wantToolCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "calculate",+ Arguments: api.ToolCallFunctionArguments{+ "expression": "2+2",+ },+ },+ },+ },+ },+ },+ },+ }++ anyOnlies := false+ for _, tc := range testCases {+ if tc.only {+ anyOnlies = true+ }+ }++ for _, tc := range testCases {+ if anyOnlies && !tc.only {+ continue+ }++ t.Run(tc.name, func(t *testing.T) {+ var chunks []api.ChatResponse+ chunkIdx := 0++ mockResponses := make([]llm.CompletionResponse, len(tc.steps))+ for i, step := range tc.steps {+ mockResponses[i] = step.input+ }++ mock := mockRunner{+ CompletionFn: func(ctx context.Context, r llm.CompletionRequest, fn func(llm.CompletionResponse)) error {+ for _, resp := range mockResponses {+ fn(resp)+ // Give the handler time to process each response+ time.Sleep(30 * time.Millisecond)+ }+ return nil+ },+ }++ s := Server{+ sched: &Scheduler{+ pendingReqCh: make(chan *LlmRequest, 1),+ finishedReqCh: make(chan *LlmRequest, 1),+ expiredCh: make(chan *runnerRef, 1),+ unloadedCh: make(chan any, 1),+ loaded: make(map[string]*runnerRef),+ newServerFn: newMockServer(&mock),+ getGpuFn: discover.GetGPUInfo,+ getCpuFn: discover.GetCPUInfo,+ reschedDelay: 100 * time.Millisecond,+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ int) {+ req.successCh <- &runnerRef{+ llama: &mock,+ }+ },+ },+ }++ go s.sched.Run(t.Context())++ // Create a simple test model+ _, digest := createHarmonyTestModel(t)++ streamFalse := false+ w := createRequest(t, s.CreateHandler, api.CreateRequest{+ Model: "harmony-test-streaming",+ Files: map[string]string{"test.gguf": digest},+ Template: `<|start|><|end|>{{ with .Tools }}{{ end }}{{ .Prompt }}`,+ Stream: &streamFalse,+ })++ if w.Code != 200 {+ t.Fatalf("failed to create model: %d", w.Code)+ }++ // Test chat endpoint with streaming+ streamTrue := true+ w = createRequest(t, s.ChatHandler, api.ChatRequest{+ Model: "harmony-test-streaming",+ Messages: []api.Message{{Role: "user", Content: "Hello"}},+ Stream: &streamTrue,+ Tools: getTestTools(),+ })++ if w.Code != 200 {+ t.Fatalf("chat request failed: %d - %s", w.Code, w.Body.String())+ }++ // Parse all chunks+ decoder := json.NewDecoder(w.Body)+ for decoder.More() {+ var chunk api.ChatResponse+ if err := decoder.Decode(&chunk); err != nil {+ t.Fatalf("failed to decode chunk: %v", err)+ }+ if chunk.Message.Content != "" || chunk.Message.Thinking != "" || len(chunk.Message.ToolCalls) > 0 {+ chunks = append(chunks, chunk)+ }+ }++ // Log received chunks for debugging+ if t.Failed() || len(chunks) == 0 {+ t.Logf("Received %d chunks:", len(chunks))+ for i, chunk := range chunks {+ t.Logf(" Chunk %d: content=%q thinking=%q", i, chunk.Message.Content, chunk.Message.Thinking)+ }+ }++ // Verify chunks match expected steps+ for i, step := range tc.steps {+ // Skip steps that don't expect any output+ if step.wantContent == "" && step.wantThinking == "" && len(step.wantToolCalls) == 0 {+ continue+ }++ if chunkIdx >= len(chunks) {+ t.Errorf("step %d: expected chunk not received (wanted content=%q thinking=%q)",+ i, step.wantContent, step.wantThinking)+ continue+ }++ chunk := chunks[chunkIdx]+ if chunk.Message.Content != step.wantContent || chunk.Message.Thinking != step.wantThinking {+ t.Errorf("step %d: chunk mismatch: got (content=%q, thinking=%q), want (content=%q, thinking=%q)",+ i, chunk.Message.Content, chunk.Message.Thinking, step.wantContent, step.wantThinking)+ }++ // Check tool calls if expected+ if len(step.wantToolCalls) > 0 {+ if len(chunk.Message.ToolCalls) != len(step.wantToolCalls) {+ t.Errorf("step %d: tool calls count mismatch: got %d, want %d",+ i, len(chunk.Message.ToolCalls), len(step.wantToolCalls))+ } else {+ for j, wantCall := range step.wantToolCalls {+ if j >= len(chunk.Message.ToolCalls) {+ break+ }+ gotCall := chunk.Message.ToolCalls[j]+ if gotCall.Function.Name != wantCall.Function.Name {+ t.Errorf("step %d, tool call %d: name mismatch: got %q, want %q",+ i, j, gotCall.Function.Name, wantCall.Function.Name)+ }+ // Compare arguments as JSON strings for simplicity+ gotArgs, _ := json.Marshal(gotCall.Function.Arguments)+ wantArgs, _ := json.Marshal(wantCall.Function.Arguments)+ if string(gotArgs) != string(wantArgs) {+ t.Errorf("step %d, tool call %d: arguments mismatch: got %s, want %s",+ i, j, string(gotArgs), string(wantArgs))+ }+ }+ }+ }+ chunkIdx+++ }++ // Check if we have extra chunks+ if chunkIdx < len(chunks) {+ t.Errorf("received %d extra chunks", len(chunks)-chunkIdx)+ for i := chunkIdx; i < len(chunks); i++ {+ t.Logf(" extra chunk %d: content=%q thinking=%q",+ i-chunkIdx, chunks[i].Message.Content, chunks[i].Message.Thinking)+ }+ }+ })+ }+}++// TestChatHarmonyParserStreamingSimple is a simpler test that just verifies basic streaming+func TestChatHarmonyParserStreamingSimple(t *testing.T) {+ gin.SetMode(gin.TestMode)++ mockResponses := []llm.CompletionResponse{+ {Content: "<|message|>First ", Done: false},+ {Content: "chunk ", Done: false},+ {Content: "here<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ }++ mock := mockRunner{+ CompletionFn: func(ctx context.Context, r llm.CompletionRequest, fn func(llm.CompletionResponse)) error {+ t.Logf("Mock received prompt: %q", r.Prompt)+ t.Logf("Mock sending %d responses", len(mockResponses))+ for i, resp := range mockResponses {+ t.Logf("Sending response %d: %q", i, resp.Content)+ fn(resp)+ }+ return nil+ },+ }++ s := Server{+ sched: &Scheduler{+ pendingReqCh: make(chan *LlmRequest, 1),+ finishedReqCh: make(chan *LlmRequest, 1),+ expiredCh: make(chan *runnerRef, 1),+ unloadedCh: make(chan any, 1),+ loaded: make(map[string]*runnerRef),+ newServerFn: newMockServer(&mock),+ getGpuFn: discover.GetGPUInfo,+ getCpuFn: discover.GetCPUInfo,+ reschedDelay: 100 * time.Millisecond,+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ int) {+ req.successCh <- &runnerRef{+ llama: &mock,+ }+ },+ },+ }++ go s.sched.Run(t.Context())++ // Create model+ _, digest := createHarmonyTestModel(t)+ streamFalse := false+ w := createRequest(t, s.CreateHandler, api.CreateRequest{+ Model: "gpt-oss",+ Files: map[string]string{"test.gguf": digest},+ Template: `<|start|><|end|>{{ .Tools }}{{ .Prompt }}`,+ Stream: &streamFalse,+ })++ if w.Code != 200 {+ t.Fatalf("failed to create model: %d", w.Code)+ }++ // Test streaming+ streamTrue := true+ w = createRequest(t, s.ChatHandler, api.ChatRequest{+ Model: "gpt-oss",+ Messages: []api.Message{{Role: "user", Content: "Hello"}},+ Stream: &streamTrue,+ Tools: getTestTools(),+ })++ if w.Code != 200 {+ t.Fatalf("chat request failed: %d - %s", w.Code, w.Body.String())+ }++ // Parse chunks+ var chunks []api.ChatResponse+ decoder := json.NewDecoder(w.Body)+ for decoder.More() {+ var chunk api.ChatResponse+ if err := decoder.Decode(&chunk); err != nil {+ t.Fatalf("failed to decode chunk: %v", err)+ }+ chunks = append(chunks, chunk)+ t.Logf("Received chunk %d: content=%q thinking=%q done=%v",+ len(chunks), chunk.Message.Content, chunk.Message.Thinking, chunk.Done)+ }++ // Verify we got chunks+ if len(chunks) == 0 {+ t.Fatal("expected streaming chunks, got none")+ }++ // Verify content+ var content strings.Builder+ for _, chunk := range chunks {+ content.WriteString(chunk.Message.Content)+ }++ expectedContent := "First chunk here"+ if content.String() != expectedContent {+ t.Errorf("content mismatch: got %q, want %q", content.String(), expectedContent)+ }++ // Verify we got multiple chunks (streaming)+ contentChunks := 0+ for _, chunk := range chunks {+ if chunk.Message.Content != "" {+ contentChunks+++ }+ }++ if contentChunks < 2 {+ t.Errorf("expected at least 2 content chunks for streaming, got %d", contentChunks)+ }+}++func TestChatHarmonyParserStreaming(t *testing.T) {+ gin.SetMode(gin.TestMode)++ type expectedChunk struct {+ afterResponse int // Which mock response this chunk should appear after+ content string // Expected content in this chunk+ thinking string // Expected thinking in this chunk+ }++ testCases := []struct {+ name string+ mockResponses []llm.CompletionResponse+ expectedChunks []expectedChunk+ wantContent string+ wantThinking string+ }{+ {+ name: "simple message without thinking",+ mockResponses: []llm.CompletionResponse{+ {Content: "<|start|>assistant<|message|>Hello, ", Done: false},+ {Content: "how can I help?", Done: false},+ {Content: "<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ },+ expectedChunks: []expectedChunk{+ {afterResponse: 1, content: "Hello, "},+ {afterResponse: 2, content: "how can I help?"},+ },+ wantContent: "Hello, how can I help?",+ },+ {+ name: "message with analysis channel for thinking",+ mockResponses: []llm.CompletionResponse{+ {Content: "<|channel|>analysis<|message|>", Done: false},+ {Content: "Let me think ", Done: false},+ {Content: "about this problem...", Done: false},+ {Content: "<|end|>", Done: false},+ {Content: "<|start|>assistant<|message|>", Done: false},+ {Content: "The answer ", Done: false},+ {Content: "is 42", Done: false},+ {Content: "<|end|>", Done: true, DoneReason: llm.DoneReasonStop},+ },+ expectedChunks: []expectedChunk{+ {afterResponse: 2, thinking: "Let me think "},+ {afterResponse: 3, thinking: "about this problem..."},+ {afterResponse: 6, content: "The answer "},+ {afterResponse: 7, content: "is 42"},+ },+ wantContent: "The answer is 42",+ wantThinking: "Let me think about this problem...",+ },+ {+ name: "streaming with partial tags across boundaries",+ mockResponses: []llm.CompletionResponse{+ {Content: "<|chan", Done: false},+ {Content: "nel|>analy", Done: false},+ {Content: "sis<|mess", Done: false},+ {Content: "age|>Think", Done: false},+ {Content: "ing deeply...<|end|>", Done: false},+ {Content: "<|start|>assi", Done: false},+ {Content: "stant<|message|>Result ", Done: false},+ {Content: "computed<|e", Done: false},+ {Content: "nd|>", Done: true, DoneReason: llm.DoneReasonStop},+ },+ expectedChunks: []expectedChunk{+ {afterResponse: 4, thinking: "Think"},+ {afterResponse: 5, thinking: "ing deeply..."},+ {afterResponse: 7, content: "Result "},+ {afterResponse: 8, content: "computed"},+ },+ wantContent: "Result computed",+ wantThinking: "Thinking deeply...",+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ // Channel to synchronize mock responses with chunk verification+ responsesSent := make(chan int, len(tc.mockResponses))++ mock := mockRunner{+ CompletionFn: func(ctx context.Context, r llm.CompletionRequest, fn func(llm.CompletionResponse)) error {+ // Send mock responses one at a time, notifying when each is sent+ for i, resp := range tc.mockResponses {+ fn(resp)+ responsesSent <- i + 1+ }+ close(responsesSent)+ return nil+ },+ }++ s := Server{+ sched: &Scheduler{+ pendingReqCh: make(chan *LlmRequest, 1),+ finishedReqCh: make(chan *LlmRequest, 1),+ expiredCh: make(chan *runnerRef, 1),+ unloadedCh: make(chan any, 1),+ loaded: make(map[string]*runnerRef),+ newServerFn: newMockServer(&mock),+ getGpuFn: discover.GetGPUInfo,+ getCpuFn: discover.GetCPUInfo,+ reschedDelay: 250 * time.Millisecond,+ loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ int) {+ req.successCh <- &runnerRef{+ llama: &mock,+ }+ },+ },+ }++ go s.sched.Run(t.Context())++ // Create a minimal model+ _, digest := createHarmonyTestModel(t)++ // Create model with passthrough template+ stream := false+ w := createRequest(t, s.CreateHandler, api.CreateRequest{+ Model: "harmony-test",+ Files: map[string]string{"file.gguf": digest},+ Template: `<|start|><|end|>{{ with .Tools }}{{ end }}{{ .Prompt }}`,+ Stream: &stream,+ })++ if w.Code != http.StatusOK {+ t.Fatalf("failed to create model: %d", w.Code)+ }++ // Test chat endpoint with streaming+ streamTrue := true+ w = createRequest(t, s.ChatHandler, api.ChatRequest{+ Model: "harmony-test",+ Messages: []api.Message{{Role: "user", Content: "Hello"}},+ Stream: &streamTrue,+ Tools: getTestTools(),+ })++ if w.Code != http.StatusOK {+ t.Fatalf("chat request failed: %d - %s", w.Code, w.Body.String())+ }++ // Parse streaming response+ var chunks []api.ChatResponse+ var content, thinking strings.Builder++ decoder := json.NewDecoder(w.Body)+ for decoder.More() {+ var chunk api.ChatResponse+ if err := decoder.Decode(&chunk); err != nil {+ t.Fatalf("failed to decode chunk: %v", err)+ }+ chunks = append(chunks, chunk)++ // Accumulate content and thinking from each chunk+ content.WriteString(chunk.Message.Content)+ thinking.WriteString(chunk.Message.Thinking)++ // Debug output+ t.Logf("Chunk %d: content=%q thinking=%q done=%v", len(chunks), chunk.Message.Content, chunk.Message.Thinking, chunk.Done)+ }++ // Verify we got streaming chunks+ if len(chunks) == 0 {+ t.Fatal("expected streaming chunks, got none")+ }++ gotContent := content.String()+ gotThinking := thinking.String()++ if gotContent != tc.wantContent {+ t.Errorf("content mismatch: got %q, want %q", gotContent, tc.wantContent)+ }+ if gotThinking != tc.wantThinking {+ t.Errorf("thinking mismatch: got %q, want %q", gotThinking, tc.wantThinking)+ }++ // Verify last chunk has done=true+ lastChunk := chunks[len(chunks)-1]+ if !lastChunk.Done {+ t.Error("expected last chunk to have done=true")+ }+ })+ }+}diff --git a/template/template.go b/template/template.goindex d28ace413..bfd02a92d 100644--- a/template/template.go+++ b/template/template.go@@ -13,6 +13,7 @@ import ("sync""text/template""text/template/parse"+ "time""github.com/agnivade/levenshtein"@@ -121,6 +122,11 @@ var funcs = template.FuncMap{b, _ := json.Marshal(v)return string(b)},+ "currentDate": func(args ...string) string {+ // Currently ignoring the format argument, but accepting it for future use+ // Default format is YYYY-MM-DD+ return time.Now().Format("2006-01-02")+ },}func Parse(s string) (*Template, error) {@@ -160,12 +166,18 @@ func (t *Template) Vars() []string {return slices.Sorted(maps.Keys(set))}+func (t *Template) Contains(s string) bool {+ return strings.Contains(t.raw, s)+}+type Values struct {Messages []api.Messageapi.ToolsPrompt stringSuffix stringThink bool+ // ThinkLevel contains the thinking level if Think is true and a string value was provided+ ThinkLevel string// whether or not the user explicitly set the thinking flag (vs. it being// implicitly false). Templates can't see whether `Think` is nilIsThinkSet bool@@ -228,6 +240,7 @@ func (t *Template) Execute(w io.Writer, v Values) error {"Suffix": v.Suffix,"Response": "","Think": v.Think,+ "ThinkLevel": v.ThinkLevel,"IsThinkSet": v.IsThinkSet,})} else if !v.forceLegacy && slices.Contains(t.Vars(), "messages") {@@ -237,6 +250,7 @@ func (t *Template) Execute(w io.Writer, v Values) error {"Tools": v.Tools,"Response": "","Think": v.Think,+ "ThinkLevel": v.ThinkLevel,"IsThinkSet": v.IsThinkSet,})}@@ -251,6 +265,7 @@ func (t *Template) Execute(w io.Writer, v Values) error {"Prompt": prompt,"Response": response,"Think": v.Think,+ "ThinkLevel": v.ThinkLevel,"IsThinkSet": v.IsThinkSet,}); err != nil {return err@@ -298,6 +313,7 @@ func (t *Template) Execute(w io.Writer, v Values) error {"Prompt": prompt,"Response": response,"Think": v.Think,+ "ThinkLevel": v.ThinkLevel,"IsThinkSet": v.IsThinkSet,}); err != nil {return errdiff --git a/tools/tools.go b/tools/tools.goindex f473ab6a6..f9ca15530 100644--- a/tools/tools.go+++ b/tools/tools.go@@ -26,6 +26,10 @@ type Parser struct {n int}+func (p *Parser) GetBuffer() []byte {+ return p.buffer+}+// NewParser creates a new tool call parser from a model's chat// template and a list of provided tools.func NewParser(tmpl *template.Template, tools []api.Tool) *Parser {