Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Qwen3VL Cloud Parser and Renderer (#12526)
Grace05982a95cb9e
diff --git a/api/types.go b/api/types.goindex 41b490b51..85a488776 100644--- a/api/types.go+++ b/api/types.go@@ -266,9 +266,9 @@ func (pt PropertyType) String() string {type ToolProperty struct {AnyOf []ToolProperty `json:"anyOf,omitempty"`- Type PropertyType `json:"type"`+ Type PropertyType `json:"type,omitempty"`Items any `json:"items,omitempty"`- Description string `json:"description"`+ Description string `json:"description,omitempty"`Enum []any `json:"enum,omitempty"`}@@ -332,7 +332,7 @@ func (t *ToolFunctionParameters) String() string {type ToolFunction struct {Name string `json:"name"`- Description string `json:"description"`+ Description string `json:"description,omitempty"`Parameters ToolFunctionParameters `json:"parameters"`}diff --git a/model/parsers/parsers.go b/model/parsers/parsers.goindex a1d4e8127..040c2562a 100644--- a/model/parsers/parsers.go+++ b/model/parsers/parsers.go@@ -21,6 +21,9 @@ func ParserForName(name string) Parser {case "qwen3-coder":parser := &Qwen3CoderParser{}return parser+ case "qwen3-vl-instruct":+ parser := &Qwen3VLParser{hasThinkingSupport: false}+ return parsercase "passthrough":return &PassthroughParser{}case "harmony":diff --git a/model/parsers/qwen3coder.go b/model/parsers/qwen3coder.goindex f44d7c8ef..84636bdc6 100644--- a/model/parsers/qwen3coder.go+++ b/model/parsers/qwen3coder.go@@ -150,7 +150,9 @@ func eat(p *Qwen3CoderParser) ([]qwenEvent, bool) {ambiguous := p.acc.String()[ambiguousStart:]p.acc.Reset()p.acc.WriteString(ambiguous)- events = append(events, qwenEventContent{content: unambiguous})+ if len(unambiguous) > 0 {+ events = append(events, qwenEventContent{content: unambiguous})+ }return events, false} else {// we found content that is entirely not a tool call. We should withholddiff --git a/model/parsers/qwen3coder_test.go b/model/parsers/qwen3coder_test.goindex c77fe2d95..3756b4bac 100644--- a/model/parsers/qwen3coder_test.go+++ b/model/parsers/qwen3coder_test.go@@ -103,6 +103,21 @@ func TestQwenParserStreaming(t *testing.T) {},},},+ {+ desc: "unambiguous empty: partial tool open at buffer start",+ steps: []step{+ {+ input: "<tool_ca",+ wantEvents: []qwenEvent{},+ },+ {+ input: "ll>abc</tool_call>",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "abc"},+ },+ },+ },+ },{desc: "trailing whitespace between tool call and content",steps: []step{diff --git a/model/parsers/qwen3vl.go b/model/parsers/qwen3vl.gonew file mode 100644index 000000000..965e32464--- /dev/null+++ b/model/parsers/qwen3vl.go@@ -0,0 +1,228 @@+package parsers++import (+ "context"+ "encoding/json"+ "log/slog"+ "strings"+ "unicode"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/logutil"+)++// TODO: call the init function+const (+ CollectingThinkingContent qwenParserState = iota+ CollectingContent+ CollectingToolContent+)++const (+ thinkingCloseTag = "</think>"+)++// TODO(gguo): add a field for isThinking+type Qwen3VLParser struct {+ state qwenParserState+ buffer strings.Builder+ tools []api.Tool+ hasThinkingSupport bool+}++func (p *Qwen3VLParser) HasToolSupport() bool {+ return true+}++// TODO(gguo): changes this to reference an objects param+func (p *Qwen3VLParser) HasThinkingSupport() bool {+ return p.hasThinkingSupport+}++func (p *Qwen3VLParser) initialState() qwenParserState {+ if p.HasThinkingSupport() { // has thinking, start from collecting thinking content+ return CollectingThinkingContent+ }+ return CollectingContent+}++func (p *Qwen3VLParser) Init(tools []api.Tool, lastMessage *api.Message) []api.Tool {+ p.tools = tools+ p.state = p.initialState()+ return tools+}++type qwenEventThinkingContent struct {+ content string+}++func (qwenEventThinkingContent) isQwenEvent() {}++func (p *Qwen3VLParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) {+ p.buffer.WriteString(s)+ events := p.parseEvents()++ var toolCalls []api.ToolCall+ var sb strings.Builder+ for _, event := range events {+ switch event := event.(type) {+ case qwenEventRawToolCall:+ toolCall, err := parseJSONToolCall(event, p.tools)+ if err != nil {+ slog.Warn("qwen tool call parsing failed", "error", err)+ return "", "", nil, err+ }+ toolCalls = append(toolCalls, toolCall)+ case qwenEventThinkingContent:+ sb.WriteString(event.content)+ case qwenEventContent:+ // TODO(drifkin): if the same turn contains multiple interleaved content+ // events, we naively append them together here.+ sb.WriteString(event.content)+ }+ }++ return sb.String(), "", toolCalls, nil+}++func (p *Qwen3VLParser) parseEvents() []qwenEvent {+ var all []qwenEvent++ keepLooping := true+ for keepLooping {+ var events []qwenEvent+ events, keepLooping = p.eat()+ if len(events) > 0 {+ all = append(all, events...)+ }+ }++ if len(all) > 0 {+ slog.Log(context.TODO(), logutil.LevelTrace, "qwen events parsed", "events", all, "state", p.state, "buffer", p.buffer.String())+ }++ return all+}++func emitContentBeforeTag(p *Qwen3VLParser, events []qwenEvent, tag string) []qwenEvent {+ split := strings.SplitN(p.buffer.String(), tag, 2)+ before := split[0]+ before = strings.TrimRightFunc(before, unicode.IsSpace)+ if len(before) > 0 {+ events = append(events, qwenEventContent{content: before})+ }+ after := split[1]+ p.buffer.Reset()+ p.buffer.WriteString(after)+ return events+}++func (p *Qwen3VLParser) eat() ([]qwenEvent, bool) {+ var events []qwenEvent++ switch p.state {+ case CollectingContent:+ if strings.Contains(p.buffer.String(), toolOpenTag) {+ events = emitContentBeforeTag(p, events, toolOpenTag)+ p.state = CollectingToolContent+ return events, true+ } else if overlapLen := overlap(p.buffer.String(), toolOpenTag); overlapLen > 0 {+ beforePartialTag := p.buffer.String()[:len(p.buffer.String())-overlapLen]+ trailingWhitespaceLen := trailingWhitespaceLen(beforePartialTag)+ ambiguousStart := len(beforePartialTag) - trailingWhitespaceLen++ unambiguous := p.buffer.String()[:ambiguousStart]+ ambiguous := p.buffer.String()[ambiguousStart:]+ p.buffer.Reset()+ p.buffer.WriteString(ambiguous)+ if len(unambiguous) > 0 {+ events = append(events, qwenEventContent{content: unambiguous})+ }+ return events, false+ } else {+ whitespaceLen := trailingWhitespaceLen(p.buffer.String())+ ambiguousStart := len(p.buffer.String()) - whitespaceLen++ unambiguous := p.buffer.String()[:ambiguousStart]+ ambiguous := p.buffer.String()[ambiguousStart:]+ p.buffer.Reset()+ p.buffer.WriteString(ambiguous)+ if len(unambiguous) > 0 {+ events = append(events, qwenEventContent{content: unambiguous})+ }+ return events, false+ }+ case CollectingToolContent:+ if strings.Contains(p.buffer.String(), toolCloseTag) {+ split := strings.SplitN(p.buffer.String(), toolCloseTag, 2)+ before := split[0]+ if len(before) == 0 {+ slog.Warn("qwen tool call closing tag found but no content before it")+ }++ after := strings.TrimLeftFunc(split[1], unicode.IsSpace)+ events = append(events, qwenEventRawToolCall{raw: before})+ p.buffer.Reset()+ p.buffer.WriteString(after)+ p.state = CollectingContent+ return events, true+ } else {+ return events, false+ }+ case CollectingThinkingContent: // so we want to hip the unambiguous stuff+ if strings.Contains(p.buffer.String(), thinkingCloseTag) {+ split := strings.SplitN(p.buffer.String(), thinkingCloseTag, 2)+ before := split[0]+ if len(before) == 0 {+ slog.Warn("qwen tool call closing tag found but no content before it")+ }+ after := strings.TrimLeftFunc(split[1], unicode.IsSpace)+ if len(before) > 0 {+ events = append(events, qwenEventThinkingContent{content: before})+ }+ p.buffer.Reset()+ p.buffer.WriteString(after)+ p.state = CollectingContent+ return events, true+ } else if overlapLen := overlap(p.buffer.String(), thinkingCloseTag); overlapLen > 0 { // we see part of a close thinking tag+ beforePartialTag := p.buffer.String()[:len(p.buffer.String())-overlapLen]+ trailingWhitespaceLen := trailingWhitespaceLen(beforePartialTag)+ ambiguousStart := len(beforePartialTag) - trailingWhitespaceLen++ unambiguous := p.buffer.String()[:ambiguousStart]+ ambiguous := p.buffer.String()[ambiguousStart:]+ p.buffer.Reset()+ p.buffer.WriteString(ambiguous)+ if len(unambiguous) > 0 {+ events = append(events, qwenEventThinkingContent{content: unambiguous})+ }+ return events, false+ } else {+ whitespaceLen := trailingWhitespaceLen(p.buffer.String())+ ambiguousStart := len(p.buffer.String()) - whitespaceLen++ unambiguous := p.buffer.String()[:ambiguousStart]+ ambiguous := p.buffer.String()[ambiguousStart:]+ p.buffer.Reset()+ p.buffer.WriteString(ambiguous)+ if len(unambiguous) > 0 {+ events = append(events, qwenEventThinkingContent{content: unambiguous})+ }+ return events, false+ }+ default:+ panic("unreachable")+ }+}++func parseJSONToolCall(raw qwenEventRawToolCall, tools []api.Tool) (api.ToolCall, error) {+ var toolCallFunction api.ToolCallFunction+ if err := json.Unmarshal([]byte(raw.raw), &toolCallFunction); err != nil {+ return api.ToolCall{}, err+ }++ toolCall := api.ToolCall{}+ toolCall.Function = toolCallFunction++ return toolCall, nil+}diff --git a/model/parsers/qwen3vl_nonthinking_test.go b/model/parsers/qwen3vl_nonthinking_test.gonew file mode 100644index 000000000..743929465--- /dev/null+++ b/model/parsers/qwen3vl_nonthinking_test.go@@ -0,0 +1,655 @@+package parsers++import (+ "reflect"+ "testing"++ "github.com/ollama/ollama/api"+)++func TestQwen3VLNonThinkingParserStreaming(t *testing.T) {+ type step struct {+ input string+ wantEvents []qwenEvent+ }++ cases := []struct {+ desc string+ steps []step+ only bool+ }{+ {+ desc: "simple thinking",+ steps: []step{+ {input: "abc</think>", wantEvents: []qwenEvent{qwenEventContent{content: "abc</think>"}}},+ },+ },+ {+ desc: "simple trip thinking",+ steps: []step{+ {input: "<think>abc</think>", wantEvents: []qwenEvent{qwenEventContent{content: "<think>abc</think>"}}},+ },+ },+ {+ desc: "thinking with split tags",+ steps: []step{+ {input: "abc", wantEvents: []qwenEvent{qwenEventContent{content: "abc"}}},+ {input: "</think>", wantEvents: []qwenEvent{qwenEventContent{content: "</think>"}}},+ },+ },+ {+ desc: "multiple think tags",+ steps: []step{+ {input: "abc<think>actually, is not thinking</think>", wantEvents: []qwenEvent{qwenEventContent{content: "abc<think>actually, is not thinking</think>"}}},+ },+ },+ {+ desc: "thinking and tool call",+ steps: []step{+ {+ input: "I'm thinking</think><tool_call>I'm tool calling</tool_call>",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "I'm thinking</think>"},+ qwenEventRawToolCall{raw: "I'm tool calling"},+ },+ },+ },+ },+ {+ desc: "nested thinking (outside thinking, inside thinking)",+ steps: []step{+ {+ input: "I'm thinking<think>I'm nested thinking</think></think>",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "I'm thinking<think>I'm nested thinking</think></think>"},+ },+ },+ },+ },+ {+ desc: "interleaved thinking",+ steps: []step{+ {+ input: "<think>I'm thinking</think>I'm actually content</think>",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "<think>I'm thinking</think>I'm actually content</think>"},+ },+ },+ },+ },+ {+ desc: "nested thinking and tool call (outside thinking, inside tool call)",+ steps: []step{+ {+ input: "I'm thinking<tool_call>I'm nested tool call</tool_call></think>",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "I'm thinking"},+ qwenEventRawToolCall{raw: "I'm nested tool call"},+ qwenEventContent{content: "</think>"},+ },+ },+ },+ },+ {+ desc: "nested thinking and tool call (outside tool call, inside thinking)",+ steps: []step{+ {+ input: "<tool_call>I'm nested tool call<think>I'm thinking</think></tool_call>",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "I'm nested tool call<think>I'm thinking</think>"},+ },+ },+ },+ },+ {+ desc: "interleaved thinking and tool call",+ steps: []step{+ {+ input: "I'm thinking<tool_call>I'm NOT a nested tool call</think></tool_call><tool_call>I'm nested tool call 2<think></tool_call></think>",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "I'm thinking"},+ qwenEventRawToolCall{raw: "I'm NOT a nested tool call</think>"},+ qwenEventRawToolCall{raw: "I'm nested tool call 2<think>"},+ qwenEventContent{content: "</think>"},+ },+ },+ },+ },+ {+ desc: "emit unambiguous before partial tool open (trailing ws)",+ steps: []step{+ {+ input: "abc\u00a0\n<tool_call",+ wantEvents: []qwenEvent{qwenEventContent{content: "abc"}},+ },+ {+ input: " fakeout",+ wantEvents: []qwenEvent{qwenEventContent{content: "\u00a0\n<tool_call fakeout"}},+ },+ },+ },+ {+ desc: "unambiguous empty: partial tool open at buffer start",+ steps: []step{+ {+ input: "<tool_ca",+ wantEvents: []qwenEvent{},+ },+ {+ input: "ll>abc</tool_call>",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "abc"},+ },+ },+ },+ },+ {+ desc: "partial thinking tag fakeout",+ steps: []step{+ {+ input: "abc</think",+ wantEvents: []qwenEvent{qwenEventContent{content: "abc</think"}},+ },+ {+ input: " fakeout",+ wantEvents: []qwenEvent{qwenEventContent{content: " fakeout"}},+ },+ },+ },+ {+ desc: "partial thinking incomplete",+ steps: []step{+ {+ input: "abc<think>unfinished<", // when something is ambiguious, we dont emit anything+ wantEvents: []qwenEvent{qwenEventContent{content: "abc<think>unfinished"}},+ },+ },+ },+ {+ desc: "test with split tool and content",+ steps: []step{+ {+ input: "abc<tool_call>unfinished</", // when something is ambiguious, we dont emit anything+ wantEvents: []qwenEvent{+ qwenEventContent{content: "abc"},+ },+ },+ {+ input: "tool_call> def",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "unfinished"},+ qwenEventContent{content: "def"},+ },+ },+ },+ },+ }+ anyOnlies := false+ for _, tc := range cases {+ if tc.only {+ anyOnlies = true+ }+ }++ for _, tc := range cases {+ if anyOnlies && !tc.only {+ continue+ }++ t.Run(tc.desc, func(t *testing.T) {+ parser := Qwen3VLParser{hasThinkingSupport: false}+ parser.Init([]api.Tool{}, nil)++ for i, step := range tc.steps {+ parser.buffer.WriteString(step.input)+ gotEvents := parser.parseEvents()++ if len(gotEvents) == 0 && len(step.wantEvents) == 0 {+ // avoid deep equal on empty vs. nil slices+ continue+ }++ if !reflect.DeepEqual(gotEvents, step.wantEvents) {+ t.Errorf("step %d: input %q: got events %#v, want %#v", i, step.input, gotEvents, step.wantEvents)+ }+ }+ })+ }+}++func TestQwenOldParserStreaming(t *testing.T) {+ type step struct {+ input string+ wantEvents []qwenEvent+ }++ cases := []struct {+ desc string+ steps []step+ only bool+ }{+ {+ desc: "simple message streamed word by word",+ steps: []step{+ {+ input: "hi",+ wantEvents: []qwenEvent{qwenEventContent{content: "hi"}},+ },+ {+ input: " there",+ wantEvents: []qwenEvent{qwenEventContent{content: " there"}},+ },+ },+ },+ {+ desc: "content before tool call",+ steps: []step{+ {+ input: "hi there<tool_call>",+ wantEvents: []qwenEvent{qwenEventContent{content: "hi there"}},+ },+ },+ },+ {+ desc: "multiple tool calls in one message",+ steps: []step{+ {+ input: "before1<tool_call>in tool call</tool_call>after1<tool_call>in tool call 2</tool_call>after2",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "before1"},+ qwenEventRawToolCall{raw: "in tool call"},+ qwenEventContent{content: "after1"},+ qwenEventRawToolCall{raw: "in tool call 2"},+ qwenEventContent{content: "after2"},+ },+ },+ },+ },+ {+ desc: "tool calls with split tags",+ steps: []step{+ {+ input: "before<tool",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "before"},+ },+ },+ {+ input: "_call>in tool call</tool",+ wantEvents: []qwenEvent{},+ },+ {+ input: "_call>af",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "in tool call"},+ qwenEventContent{content: "af"},+ },+ },+ {+ input: "ter",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "ter"},+ },+ },+ },+ },+ {+ desc: "trailing whitespace between content and tool call",+ steps: []step{+ {+ input: "abc\n<tool_call>def</tool_call>",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "abc"},+ qwenEventRawToolCall{raw: "def"},+ },+ },+ },+ },+ {+ desc: "trailing whitespace between tool call and content",+ steps: []step{+ {+ input: "<tool_call>abc</tool_call>\ndef",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "abc"},+ qwenEventContent{content: "def"},+ },+ },+ },+ },+ {+ desc: "empty content before tool call",+ steps: []step{+ {+ input: "\n<tool_call>abc</tool_call>",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "abc"},+ },+ },+ },+ },+ {+ desc: "partial tool open tag fakeout",+ steps: []step{+ {+ input: "abc\n<tool_call",+ wantEvents: []qwenEvent{+ // \n should not be emitted yet because `<tool_call` might be a tool+ // open tag, in which case the whitespace should be trimmed+ qwenEventContent{content: "abc"},+ },+ },+ {+ input: " fakeout",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "\n<tool_call fakeout"},+ },+ },+ },+ },+ {+ desc: "token-by-token whitespace handling",+ steps: []step{+ {+ input: "a",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "a"},+ },+ },+ {+ input: "\n",+ wantEvents: []qwenEvent{},+ },+ {+ input: "b",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "\nb"},+ },+ },+ },+ },+ {+ desc: "unicode content",+ steps: []step{+ {+ input: "你好 🌍<tool_call>test</tool_call>مرحبا",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "你好 🌍"},+ qwenEventRawToolCall{raw: "test"},+ qwenEventContent{content: "مرحبا"},+ },+ },+ },+ },+ {+ desc: "arabic text handling",+ steps: []step{+ {+ input: "مرحبا بالعالم",+ wantEvents: []qwenEvent{qwenEventContent{content: "مرحبا بالعالم"}},+ },+ },+ },+ {+ desc: "emoji passthrough",+ steps: []step{+ {+ input: "✅",+ wantEvents: []qwenEvent{qwenEventContent{content: "✅"}},+ },+ },+ },+ {+ desc: "emoji after tool call",+ steps: []step{+ {+ input: "<tool_call>test</tool_call>完成 ✅",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "test"},+ qwenEventContent{content: "完成 ✅"},+ },+ },+ },+ },+ {+ desc: "unicode streaming with whitespace handling",+ steps: []step{+ {+ input: "مرحبا",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "مرحبا"},+ },+ },+ {+ input: " \n",+ wantEvents: []qwenEvent{},+ },+ {+ input: "世界",+ wantEvents: []qwenEvent{+ qwenEventContent{content: " \n世界"},+ },+ },+ },+ },+ {+ desc: "non-breaking space withheld across chunks",+ steps: []step{+ {+ input: "Hello\u00a0",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "Hello"},+ },+ },+ {+ input: "world",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "\u00a0world"},+ },+ },+ },+ },+ {+ desc: "ideographic space before partial tool",+ steps: []step{+ {+ input: "Hello\u3000<tool",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "Hello"},+ },+ },+ {+ input: "_call>abc",+ wantEvents: []qwenEvent{},+ },+ {+ input: "</tool_call>def",+ wantEvents: []qwenEvent{+ qwenEventRawToolCall{raw: "abc"},+ qwenEventContent{content: "def"},+ },+ },+ },+ },+ {+ desc: "ideographic space before partial tool fakeout",+ steps: []step{+ {+ input: "Hello\u3000<tool",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "Hello"},+ },+ },+ {+ input: "fakeout>abc",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "\u3000<toolfakeout>abc"},+ },+ },+ },+ },+ {+ desc: "unicode with partial tool tag",+ steps: []step{+ {+ input: "测试🎯 <to",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "测试🎯"},+ },+ },+ },+ },+ }++ anyOnlies := false+ for _, tc := range cases {+ if tc.only {+ anyOnlies = true+ }+ }++ for _, tc := range cases {+ if anyOnlies && !tc.only {+ continue+ }++ t.Run(tc.desc, func(t *testing.T) {+ parser := Qwen3VLParser{hasThinkingSupport: false}+ parser.Init([]api.Tool{}, nil)++ for i, step := range tc.steps {+ parser.buffer.WriteString(step.input)+ gotEvents := parser.parseEvents()++ if len(gotEvents) == 0 && len(step.wantEvents) == 0 {+ // avoid deep equal on empty vs. nil slices+ continue+ }++ if !reflect.DeepEqual(gotEvents, step.wantEvents) {+ t.Errorf("step %d: input %q: got events %#v, want %#v", i, step.input, gotEvents, step.wantEvents)+ }+ }+ })+ }+}++func TestQwen3VLNonThinkingToolParser(t *testing.T) {+ type step struct {+ name string+ rawToolCall string+ tools []api.Tool+ wantToolCall api.ToolCall+ }++ steps := []step{+ {+ name: "simple tool call",+ tools: []api.Tool{},+ rawToolCall: `{"name": "get-current-weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "get-current-weather",+ Arguments: map[string]any{+ "location": "San Francisco, CA",+ "unit": "fahrenheit",+ },+ },+ },+ },+ {+ name: "names with spaces",+ tools: []api.Tool{},+ rawToolCall: `{"name": "get current temperature", "arguments": {"location with spaces": "San Francisco", "unit with spaces": "celsius"}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "get current temperature",+ Arguments: map[string]any{+ "location with spaces": "San Francisco",+ "unit with spaces": "celsius",+ },+ },+ },+ },+ {+ name: "names with quotes",+ tools: []api.Tool{},+ rawToolCall: `{"name": "\"get current temperature\"", "arguments": {"\"location with spaces\"": "San Francisco", "\"unit with spaces\"": "\"celsius\""}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "\"get current temperature\"",+ Arguments: map[string]any{+ "\"location with spaces\"": "San Francisco",+ "\"unit with spaces\"": "\"celsius\"",+ },+ },+ },+ },+ {+ name: "tool call with typed parameters (json types)",+ tools: []api.Tool{},+ rawToolCall: `{"name": "calculate", "arguments": {"x": 3.14, "y": 42, "enabled": true, "items": ["a", "b", "c"]}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "calculate",+ Arguments: map[string]any{+ "x": 3.14,+ "y": float64(42),+ "enabled": true,+ "items": []any{"a", "b", "c"},+ },+ },+ },+ },+ {+ name: "ampersands in parameter values",+ tools: []api.Tool{},+ rawToolCall: `{"name": "exec", "arguments": {"command": "ls && echo \"done\""}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "exec",+ Arguments: map[string]any{+ "command": "ls && echo \"done\"",+ },+ },+ },+ },+ {+ name: "angle brackets in parameter values",+ tools: []api.Tool{},+ rawToolCall: `{"name": "exec", "arguments": {"command": "ls && echo \"a > b and a < b\""}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "exec",+ Arguments: map[string]any{+ "command": "ls && echo \"a > b and a < b\"",+ },+ },+ },+ },+ {+ name: "unicode in function names and parameters",+ tools: []api.Tool{},+ rawToolCall: `{"name": "获取天气", "arguments": {"城市": "北京", "message": "Hello! 你好! 🌟 مرحبا"}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "获取天气",+ Arguments: map[string]any{+ "城市": "北京",+ "message": "Hello! 你好! 🌟 مرحبا",+ },+ },+ },+ },+ }++ for i, step := range steps {+ gotToolCall, err := parseJSONToolCall(qwenEventRawToolCall{raw: step.rawToolCall}, step.tools)+ if err != nil {+ t.Errorf("step %d (%s): %v", i, step.name, err)+ }+ if !reflect.DeepEqual(gotToolCall, step.wantToolCall) {+ t.Errorf("step %d (%s): got tool call %#v, want %#v", i, step.name, gotToolCall, step.wantToolCall)+ }+ }+}diff --git a/model/parsers/qwen3vl_thinking_test.go b/model/parsers/qwen3vl_thinking_test.gonew file mode 100644index 000000000..a94344a4f--- /dev/null+++ b/model/parsers/qwen3vl_thinking_test.go@@ -0,0 +1,346 @@+package parsers++import (+ "reflect"+ "testing"++ "github.com/ollama/ollama/api"+)++func TestQwen3VLThinkingParserStreaming(t *testing.T) {+ type step struct {+ input string+ wantEvents []qwenEvent+ }++ cases := []struct {+ desc string+ steps []step+ only bool+ }{+ {+ desc: "simple thinking",+ steps: []step{+ {input: "abc</think>", wantEvents: []qwenEvent{qwenEventThinkingContent{content: "abc"}}},+ },+ },+ {+ desc: "simple trip thinking",+ steps: []step{+ {input: "<think>abc</think>", wantEvents: []qwenEvent{qwenEventThinkingContent{content: "<think>abc"}}},+ },+ },+ {+ desc: "thinking with split tags",+ steps: []step{+ {input: "abc", wantEvents: []qwenEvent{qwenEventThinkingContent{content: "abc"}}},+ {input: "</think>", wantEvents: []qwenEvent{}},+ },+ },+ {+ desc: "multiple think tags",+ steps: []step{+ {input: "abc<think>actually, is not thinking</think>", wantEvents: []qwenEvent{qwenEventThinkingContent{content: "abc<think>actually, is not thinking"}}},+ },+ },+ {+ desc: "thinking and tool call",+ steps: []step{+ {+ input: "I'm thinking</think><tool_call>I'm tool calling</tool_call>",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "I'm thinking"},+ qwenEventRawToolCall{raw: "I'm tool calling"},+ },+ },+ },+ },+ {+ desc: "thinking and content",+ steps: []step{+ {+ input: "I'm thinking</think>I'm content",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "I'm thinking"},+ qwenEventContent{content: "I'm content"},+ },+ },+ },+ },+ {+ desc: "thinking and tool call and content",+ },+ {+ desc: "nested thinking (outside thinking, inside thinking)",+ steps: []step{+ {+ input: "I'm thinking<think>I'm nested thinking</think></think>",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "I'm thinking<think>I'm nested thinking"},+ qwenEventContent{content: "</think>"},+ },+ },+ },+ },+ {+ desc: "interleaved thinking",+ steps: []step{+ {+ input: "<think>I'm thinking</think>I'm actually content</think>",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "<think>I'm thinking"},+ qwenEventContent{content: "I'm actually content</think>"},+ },+ },+ },+ },+ {+ desc: "nested thinking and tool call (outside thinking, inside tool call)",+ steps: []step{+ {+ input: "I'm thinking<tool_call>I'm nested tool call</tool_call></think>",+ wantEvents: []qwenEvent{qwenEventThinkingContent{content: "I'm thinking<tool_call>I'm nested tool call</tool_call>"}},+ },+ },+ },+ {+ desc: "nested thinking and tool call (outside tool call, inside thinking)",+ steps: []step{+ {+ input: "<tool_call>I'm nested tool call<think>I'm thinking</think></tool_call>",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "<tool_call>I'm nested tool call<think>I'm thinking"},+ qwenEventContent{content: "</tool_call>"},+ },+ },+ },+ },+ {+ desc: "interleaved thinking and tool call",+ steps: []step{+ {+ input: "I'm thinking<tool_call>I'm NOT a nested tool call</think></tool_call><tool_call>I'm nested tool call 2<think></tool_call></think>",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "I'm thinking<tool_call>I'm NOT a nested tool call"},+ qwenEventContent{content: "</tool_call>"},+ qwenEventRawToolCall{raw: "I'm nested tool call 2<think>"},+ qwenEventContent{content: "</think>"},+ },+ },+ },+ },+ {+ desc: "partial thinking tag fakeout",+ steps: []step{+ {+ input: "abc</think",+ wantEvents: []qwenEvent{qwenEventThinkingContent{content: "abc"}},+ },+ {+ input: " fakeout",+ wantEvents: []qwenEvent{qwenEventThinkingContent{content: "</think fakeout"}},+ },+ },+ },+ {+ desc: "partial thinking incomplete",+ steps: []step{+ {+ input: "abc<think>unfinished</think", // when something is ambiguious, we dont emit anything+ wantEvents: []qwenEvent{qwenEventThinkingContent{content: "abc<think>unfinished"}},+ },+ },+ },+ {+ desc: "test with split thinking and content",+ steps: []step{+ {+ input: "abc<think>unfinished</th", // when something is ambiguious, we dont emit anything+ wantEvents: []qwenEvent{qwenEventThinkingContent{content: "abc<think>unfinished"}},+ },+ {+ input: "ink> def",+ wantEvents: []qwenEvent{+ qwenEventContent{content: "def"},+ },+ },+ },+ },+ {+ desc: "thinking with no tags",+ steps: []step{+ {+ input: "Hello I am thinking",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "Hello I am thinking"},+ },+ },+ {+ input: "Hello I am thinking some more",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "Hello I am thinking some more"},+ },+ },+ {+ input: "Hello I am think</think> NOT",+ wantEvents: []qwenEvent{+ qwenEventThinkingContent{content: "Hello I am think"},+ qwenEventContent{content: "NOT"},+ },+ },+ },+ },+ }+ anyOnlies := false+ for _, tc := range cases {+ if tc.only {+ anyOnlies = true+ }+ }++ for _, tc := range cases {+ if anyOnlies && !tc.only {+ continue+ }++ t.Run(tc.desc, func(t *testing.T) {+ parser := Qwen3VLParser{hasThinkingSupport: true}+ parser.Init([]api.Tool{}, nil)+ // parser.state = CollectingThinkingContent++ for i, step := range tc.steps {+ parser.buffer.WriteString(step.input)+ gotEvents := parser.parseEvents()++ if len(gotEvents) == 0 && len(step.wantEvents) == 0 {+ // avoid deep equal on empty vs. nil slices+ continue+ }++ if !reflect.DeepEqual(gotEvents, step.wantEvents) {+ t.Errorf("step %d: input %q: got events %#v, want %#v", i, step.input, gotEvents, step.wantEvents)+ }+ }+ })+ }+}++func TestQwen3VLThinkingToolParser(t *testing.T) {+ type step struct {+ name string+ rawToolCall string+ tools []api.Tool+ wantToolCall api.ToolCall+ }++ steps := []step{+ {+ name: "simple tool call",+ tools: []api.Tool{},+ rawToolCall: `{"name": "get-current-weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "get-current-weather",+ Arguments: map[string]any{+ "location": "San Francisco, CA",+ "unit": "fahrenheit",+ },+ },+ },+ },+ {+ name: "names with spaces",+ tools: []api.Tool{},+ rawToolCall: `{"name": "get current temperature", "arguments": {"location with spaces": "San Francisco", "unit with spaces": "celsius"}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "get current temperature",+ Arguments: map[string]any{+ "location with spaces": "San Francisco",+ "unit with spaces": "celsius",+ },+ },+ },+ },+ {+ name: "names with quotes",+ tools: []api.Tool{},+ rawToolCall: `{"name": "\"get current temperature\"", "arguments": {"\"location with spaces\"": "San Francisco", "\"unit with spaces\"": "\"celsius\""}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "\"get current temperature\"",+ Arguments: map[string]any{+ "\"location with spaces\"": "San Francisco",+ "\"unit with spaces\"": "\"celsius\"",+ },+ },+ },+ },+ {+ name: "tool call with typed parameters (json types)",+ tools: []api.Tool{},+ rawToolCall: `{"name": "calculate", "arguments": {"x": 3.14, "y": 42, "enabled": true, "items": ["a", "b", "c"]}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "calculate",+ Arguments: map[string]any{+ "x": 3.14,+ "y": float64(42),+ "enabled": true,+ "items": []any{"a", "b", "c"},+ },+ },+ },+ },+ {+ name: "ampersands in parameter values",+ tools: []api.Tool{},+ rawToolCall: `{"name": "exec", "arguments": {"command": "ls && echo \"done\""}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "exec",+ Arguments: map[string]any{+ "command": "ls && echo \"done\"",+ },+ },+ },+ },+ {+ name: "angle brackets in parameter values",+ tools: []api.Tool{},+ rawToolCall: `{"name": "exec", "arguments": {"command": "ls && echo \"a > b and a < b\""}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "exec",+ Arguments: map[string]any{+ "command": "ls && echo \"a > b and a < b\"",+ },+ },+ },+ },+ {+ name: "unicode in function names and parameters",+ tools: []api.Tool{},+ rawToolCall: `{"name": "获取天气", "arguments": {"城市": "北京", "message": "Hello! 你好! 🌟 مرحبا"}}`,+ wantToolCall: api.ToolCall{+ Function: api.ToolCallFunction{+ Name: "获取天气",+ Arguments: map[string]any{+ "城市": "北京",+ "message": "Hello! 你好! 🌟 مرحبا",+ },+ },+ },+ },+ }++ for i, step := range steps {+ gotToolCall, err := parseJSONToolCall(qwenEventRawToolCall{raw: step.rawToolCall}, step.tools)+ if err != nil {+ t.Errorf("step %d (%s): %v", i, step.name, err)+ }+ if !reflect.DeepEqual(gotToolCall, step.wantToolCall) {+ t.Errorf("step %d (%s): got tool call %#v, want %#v", i, step.name, gotToolCall, step.wantToolCall)+ }+ }+}diff --git a/model/renderers/qwen3coder.go b/model/renderers/qwen3coder.goindex 32611791b..18853019c 100644--- a/model/renderers/qwen3coder.go+++ b/model/renderers/qwen3coder.go@@ -55,7 +55,9 @@ func renderAdditionalKeys(obj any, handledKeys map[string]bool) string {return sb.String()}-func Qwen3CoderRenderer(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {+type Qwen3CoderRenderer struct{}++func (r *Qwen3CoderRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {var sb strings.Builder// filter out system messages and choose the first (if any) to windiff --git a/model/renderers/qwen3coder_test.go b/model/renderers/qwen3coder_test.goindex 6a9e5eccd..1addee9e1 100644--- a/model/renderers/qwen3coder_test.go+++ b/model/renderers/qwen3coder_test.go@@ -288,7 +288,7 @@ call tool<|im_end|>}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- rendered, err := Qwen3CoderRenderer(tt.msgs, tt.tools, nil)+ rendered, err := (&Qwen3CoderRenderer{}).Render(tt.msgs, tt.tools, nil)if err != nil {t.Fatal(err)}diff --git a/model/renderers/qwen3vl.go b/model/renderers/qwen3vl.gonew file mode 100644index 000000000..7e49fea18--- /dev/null+++ b/model/renderers/qwen3vl.go@@ -0,0 +1,166 @@+package renderers++import (+ "encoding/json"+ "strings"++ "github.com/ollama/ollama/api"+)++func marshalWithSpaces(v any) ([]byte, error) {+ b, err := json.Marshal(v)+ if err != nil {+ return nil, err+ }++ out := make([]byte, 0, len(b)+len(b)/8)+ inStr, esc := false, false+ for _, c := range b {+ if inStr {+ out = append(out, c)+ if esc {+ esc = false+ continue+ }+ if c == '\\' {+ esc = true+ continue+ }+ if c == '"' {+ inStr = false+ }+ continue+ }+ switch c {+ case '"':+ inStr = true+ out = append(out, c)+ case ':':+ out = append(out, ':', ' ')+ case ',':+ out = append(out, ',', ' ')+ default:+ out = append(out, c)+ }+ }+ return out, nil+}++type Qwen3VLRenderer struct {+ isThinking bool+}++func (r *Qwen3VLRenderer) renderContent(content api.Message, doVisionCount bool) string {+ // This assumes all images are at the front of the message - same assumption as ollama/ollama/runner.go+ var subSb strings.Builder+ for range content.Images {+ subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>")+ }+ // TODO: support videos++ subSb.WriteString(content.Content)+ return subSb.String()+}++func (r *Qwen3VLRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) {+ var sb strings.Builder++ if len(tools) > 0 {+ sb.WriteString(imStartTag + "system\n")+ if len(messages) > 0 && messages[0].Role == "system" {+ sb.WriteString(messages[0].Content + "\n\n")+ }+ sb.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>")+ for _, tool := range tools {+ sb.WriteString("\n")+ if b, err := marshalWithSpaces(tool); err == nil {+ sb.Write(b)+ }+ }+ sb.WriteString("\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n")+ } else if len(messages) > 0 && messages[0].Role == "system" {+ sb.WriteString("<|im_start|>system\n" + messages[0].Content + "<|im_end|>\n")+ }+ multiStepTool := true+ lastQueryIndex := len(messages) - 1 // so this is the last user message++ for i := len(messages) - 1; i >= 0; i-- {+ message := messages[i]+ if multiStepTool && message.Role == "user" {+ // Check if content starts with <tool_response> and ends with </tool_response>+ content := r.renderContent(message, true)+ if !(strings.HasPrefix(content, "<tool_response>") && strings.HasSuffix(content, "</tool_response>")) {+ multiStepTool = false+ lastQueryIndex = i+ }+ }+ }++ for i, message := range messages {+ content := r.renderContent(message, true)++ lastMessage := i == len(messages)-1+ prefill := lastMessage && message.Role == "assistant"++ if message.Role == "user" || message.Role == "system" && i != 0 {+ sb.WriteString("<|im_start|>" + message.Role + "\n" + content + "<|im_end|>\n")+ } else if message.Role == "assistant" {+ contentReasoning := ""++ if r.isThinking {+ if message.Thinking != "" {+ contentReasoning = message.Thinking+ }+ }++ if r.isThinking && i > lastQueryIndex {+ if i == len(messages)-1 || contentReasoning != "" {+ sb.WriteString("<|im_start|>" + message.Role + "\n<think>\n" + strings.Trim(contentReasoning, "\n")) // do we want to add a new line here?+ if content != "" {+ sb.WriteString("\n</think>\n\n" + strings.TrimLeft(content, "\n"))+ }+ } else {+ sb.WriteString("<|im_start|>" + message.Role + "\n" + content)+ }+ } else {+ sb.WriteString("<|im_start|>" + message.Role + "\n" + content)+ }++ if len(message.ToolCalls) > 0 {+ for j, toolCall := range message.ToolCalls {+ if j > 0 || content != "" {+ sb.WriteString("\n")+ }++ sb.WriteString("<tool_call>\n{\"name\": \"" + toolCall.Function.Name + "\", \"arguments\": ")+ if b, err := marshalWithSpaces(toolCall.Function.Arguments); err == nil {+ sb.Write(b)+ }+ sb.WriteString("}\n</tool_call>")+ }+ }++ if !prefill {+ sb.WriteString("<|im_end|>\n")+ }+ } else if message.Role == "tool" {+ if i == 0 || messages[i-1].Role != "tool" {+ sb.WriteString("<|im_start|>user")+ }+ sb.WriteString("\n<tool_response>\n" + message.Content + "\n</tool_response>")+ if i == len(messages)-1 || messages[i+1].Role != "tool" {+ sb.WriteString("<|im_end|>\n")+ }+ }++ // prefill at the end+ if lastMessage && !prefill {+ sb.WriteString("<|im_start|>assistant\n")+ if r.isThinking {+ sb.WriteString("<think>\n")+ }+ }+ }++ return sb.String(), nil+}diff --git a/model/renderers/qwen3vl_nonthinking_test.go b/model/renderers/qwen3vl_nonthinking_test.gonew file mode 100644index 000000000..3f50a965e--- /dev/null+++ b/model/renderers/qwen3vl_nonthinking_test.go@@ -0,0 +1,497 @@+package renderers++import (+ "testing"++ "github.com/google/go-cmp/cmp"+ "github.com/ollama/ollama/api"+)++func TestQwen3VLNonThinkingRenderer(t *testing.T) {+ tests := []struct {+ name string+ msgs []api.Message+ images []api.ImageData+ tools []api.Tool+ expected string+ }{+ {+ name: "prefill",+ msgs: []api.Message{+ {Role: "system", Content: "You are a helpful assistant."},+ {Role: "user", Content: "Tell me something interesting."},+ {Role: "assistant", Content: "I'll tell you something interesting about cats"},+ },+ expected: `<|im_start|>system+You are a helpful assistant.<|im_end|>+<|im_start|>user+Tell me something interesting.<|im_end|>+<|im_start|>assistant+I'll tell you something interesting about cats`,+ },+ {+ name: "basic",+ msgs: []api.Message{+ {Role: "system", Content: "You are a helpful assistant."},+ {Role: "user", Content: "Hello, how are you?"},+ },+ expected: `<|im_start|>system+You are a helpful assistant.<|im_end|>+<|im_start|>user+Hello, how are you?<|im_end|>+<|im_start|>assistant+`,+ },+ {+ name: "With thinking, end assistant.",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Content: "abc<think>To make this story interesting, I will speak in poetry.</think>"}, // does the thinking even work?+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+abc<think>To make this story interesting, I will speak in poetry.</think>`,+ },+ {+ name: "Multiple thinking",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Content: "abc<think>To make this story interesting, I will speak in poetry.</think><think>And I will speak in poetry after the first sentence.</think>"},+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+abc<think>To make this story interesting, I will speak in poetry.</think><think>And I will speak in poetry after the first sentence.</think>`, // NOTE: the second thinking tag is not captured+ },+ {+ name: "Multiple thinking, multiple messages.",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Content: "abc<think>To make this story interesting, I will speak in poetry.</think><think>And I will speak in poetry after the first sentence.</think>"},+ {Role: "user", Content: "What is the weather like in San Francisco? <think>I will check the weather in San Francisco for you.</think>"},+ {Role: "assistant", Content: "I'll check the weather in San Francisco for you.<think>Speak poetry after the first sentence.</think><think>Speak poetry after the second sentence.</think>"},+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+abc<think>To make this story interesting, I will speak in poetry.</think><think>And I will speak in poetry after the first sentence.</think><|im_end|>+<|im_start|>user+What is the weather like in San Francisco? <think>I will check the weather in San Francisco for you.</think><|im_end|>+<|im_start|>assistant+I'll check the weather in San Francisco for you.<think>Speak poetry after the first sentence.</think><think>Speak poetry after the second sentence.</think>`,+ },+ {+ name: "Image",+ msgs: []api.Message{+ {Role: "user", Content: "Describe this image.", Images: []api.ImageData{api.ImageData("img2")}},+ {Role: "assistant", Content: "Let me analyze this image."},+ },+ expected: `<|im_start|>user+<|vision_start|><|image_pad|><|vision_end|>Describe this image.<|im_end|>+<|im_start|>assistant+Let me analyze this image.`,+ },+ {+ name: "Multiple images",+ msgs: []api.Message{+ {Role: "user", Content: "Describe these images.", Images: []api.ImageData{api.ImageData("img1"), api.ImageData("img2")}},+ },+ expected: `<|im_start|>user+<|vision_start|><|image_pad|><|vision_end|><|vision_start|><|image_pad|><|vision_end|>Describe these images.<|im_end|>+<|im_start|>assistant+`,+ },++ // // NOTE: solved with #12518: https://github.com/ollama/ollama/compare/main...drifkin/stable-tool-args+ // {+ // name: "with tools and response",+ // msgs: []api.Message{+ // {Role: "system", Content: "You are a helpful assistant with access to tools."},+ // {Role: "user", Content: "What's the weather like in New York?"},+ // {+ // Role: "assistant",+ // Content: "I'll check the weather in New York for you.",+ // ToolCalls: []api.ToolCall{+ // {+ // Function: api.ToolCallFunction{+ // Name: "get-current-weather",+ // Arguments: map[string]any{+ // "location": "New York",+ // "unit": "fahrenheit",+ // },+ // },+ // },+ // },+ // },+ // {Role: "tool", Content: "80", ToolName: "get-current-weather"},+ // {Role: "user", Content: "That sounds nice! What about San Francisco?"},+ // },+ // tools: []api.Tool{+ // {+ // Type: "function",+ // Function: api.ToolFunction{+ // Name: "get-current-weather",+ // Description: "Get the current weather for a location",+ // Parameters: api.ToolFunctionParameters{+ // Type: "object",+ // Required: []string{"location"},+ // Properties: map[string]api.ToolProperty{+ // "location": {+ // Type: api.PropertyType{"string"},+ // Description: "The city and state, e.g. San Francisco, CA",+ // },+ // "unit": {+ // Type: api.PropertyType{"string"},+ // Enum: []any{"celsius", "fahrenheit"},+ // Description: "The temperature unit",+ // },+ // },+ // },+ // },+ // },+ // },+ // expected: `<|im_start|>system+ // You are a helpful assistant with access to tools.++ // # Tools++ // You may call one or more functions to assist with the user query.++ // You are provided with function signatures within <tools></tools> XML tags:+ // <tools>+ // {"type": "function", "function": {"name": "get-current-weather", "description": "Get the current weather for a location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "The temperature unit"}}, "required": ["location"]}}}+ // </tools>++ // For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:+ // <tool_call>+ // {"name": <function-name>, "arguments": <args-json-object>}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // What's the weather like in New York?<|im_end|>+ // <|im_start|>assistant+ // I'll check the weather in New York for you.+ // <tool_call>+ // {"name": "get-current-weather", "arguments": {"location": "New York", "unit": "fahrenheit"}}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // <tool_response>+ // 80+ // </tool_response><|im_end|>+ // <|im_start|>user+ // That sounds nice! What about San Francisco?<|im_end|>+ // <|im_start|>assistant+ // `,+ // },+ // // NOTE: solved with #12518: https://github.com/ollama/ollama/compare/main...drifkin/stable-tool-args+ // {+ // name: "With tools and response, multiple tool calls",+ // msgs: []api.Message{+ // {+ // Role: "system",+ // Content: "You are a helpful assistant with access to tools.",+ // },+ // {+ // Role: "user",+ // Content: "Call two tools for me: add and multiply.",+ // },+ // {+ // Role: "assistant",+ // Content: "Sure, I'll call both tools for you.",+ // ToolCalls: []api.ToolCall{+ // {+ // Function: api.ToolCallFunction{+ // Name: "add",+ // Arguments: map[string]any{+ // "a": 2,+ // "b": 3,+ // },+ // },+ // },+ // {+ // Function: api.ToolCallFunction{+ // Name: "multiply",+ // Arguments: map[string]any{+ // "x": 4,+ // "y": 5,+ // },+ // },+ // },+ // },+ // },+ // {+ // Role: "tool",+ // Content: "5",+ // ToolName: "add",+ // },+ // {+ // Role: "tool",+ // Content: "20",+ // ToolName: "multiply",+ // },+ // {+ // Role: "user",+ // Content: "Thanks! What are the results?",+ // },+ // },+ // tools: []api.Tool{+ // {+ // Type: "function",+ // Function: api.ToolFunction{+ // Name: "add",+ // Description: "Add two numbers",+ // Parameters: api.ToolFunctionParameters{+ // Type: "object",+ // Required: []string{"a", "b"},+ // Properties: map[string]api.ToolProperty{+ // "a": {Type: api.PropertyType{"integer"}, Description: "First number"},+ // "b": {Type: api.PropertyType{"integer"}, Description: "Second number"},+ // },+ // },+ // },+ // },+ // {+ // Type: "function",+ // Function: api.ToolFunction{+ // Name: "multiply",+ // Description: "Multiply two numbers",+ // Parameters: api.ToolFunctionParameters{+ // Type: "object",+ // Required: []string{"x", "y"},+ // Properties: map[string]api.ToolProperty{+ // "x": {Type: api.PropertyType{"integer"}, Description: "First factor"},+ // "y": {Type: api.PropertyType{"integer"}, Description: "Second factor"},+ // },+ // },+ // },+ // },+ // },+ // expected: `<|im_start|>system+ // You are a helpful assistant with access to tools.++ // # Tools++ // You may call one or more functions to assist with the user query.++ // You are provided with function signatures within <tools></tools> XML tags:+ // <tools>+ // {"type": "function", "function": {"name": "add", "description": "Add two numbers", "parameters": {"type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"]}}}+ // {"type": "function", "function": {"name": "multiply", "description": "Multiply two numbers", "parameters": {"type": "object", "properties": {"x": {"description": "First factor"}, "y": {"description": "Second factor"}}, "required": ["x", "y"]}}}+ // </tools>++ // For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:+ // <tool_call>+ // {"name": <function-name>, "arguments": <args-json-object>}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // Call two tools for me: add and multiply.<|im_end|>+ // <|im_start|>assistant+ // Sure, I'll call both tools for you.+ // <tool_call>+ // {"name": "add", "arguments": {"a": 2, "b": 3}}+ // </tool_call>+ // <tool_call>+ // {"name": "multiply", "arguments": {"x": 4, "y": 5}}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // <tool_response>+ // 5+ // </tool_response>+ // <tool_response>+ // 20+ // </tool_response><|im_end|>+ // <|im_start|>user+ // Thanks! What are the results?<|im_end|>+ // <|im_start|>assistant+ // `,+ // },+ {+ name: "user tool_response block preserved",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather?"},+ {+ Role: "assistant",+ Content: "I'll check.",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "get-current-weather", Arguments: map[string]any{"location": "Paris", "unit": "celsius"}}},+ },+ },+ {Role: "user", Content: "<tool_response>\n18\n</tool_response>"},+ {Role: "user", Content: "Thanks!"},+ },+ expected: `<|im_start|>user+What's the weather?<|im_end|>+<|im_start|>assistant+I'll check.+<tool_call>+{"name": "get-current-weather", "arguments": {"location": "Paris", "unit": "celsius"}}+</tool_call><|im_end|>+<|im_start|>user+<tool_response>+18+</tool_response><|im_end|>+<|im_start|>user+Thanks!<|im_end|>+<|im_start|>assistant+`,+ },+ {+ name: "assistant with multiple tool calls and content",+ msgs: []api.Message{+ {Role: "user", Content: "Hi"},+ {+ Role: "assistant",+ Content: "before",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "add", Arguments: map[string]any{"a": 2, "b": 3}}},+ {Function: api.ToolCallFunction{Name: "mul", Arguments: map[string]any{"x": 4, "y": 5}}},+ },+ },+ },+ expected: `<|im_start|>user+Hi<|im_end|>+<|im_start|>assistant+before+<tool_call>+{"name": "add", "arguments": {"a": 2, "b": 3}}+</tool_call>+<tool_call>+{"name": "mul", "arguments": {"x": 4, "y": 5}}+</tool_call>`,+ },+ {+ name: "consecutive tool responses grouped",+ msgs: []api.Message{+ {Role: "user", Content: "Compute results"},+ {Role: "assistant", Content: "ok", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "job", Arguments: map[string]any{"n": 1}}}}},+ {Role: "tool", Content: "5", ToolName: "job"},+ {Role: "tool", Content: "6", ToolName: "job"},+ },+ expected: `<|im_start|>user+Compute results<|im_end|>+<|im_start|>assistant+ok+<tool_call>+{"name": "job", "arguments": {"n": 1}}+</tool_call><|im_end|>+<|im_start|>user+<tool_response>+5+</tool_response>+<tool_response>+6+</tool_response><|im_end|>+<|im_start|>assistant+`,+ },+ {+ name: "last message is tool then prefill",+ msgs: []api.Message{+ {Role: "user", Content: "run"},+ {Role: "assistant", Content: "ok", ToolCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "exec", Arguments: map[string]any{"cmd": "ls"}}}}},+ {Role: "tool", Content: "done", ToolName: "exec"},+ },+ expected: `<|im_start|>user+run<|im_end|>+<|im_start|>assistant+ok+<tool_call>+{"name": "exec", "arguments": {"cmd": "ls"}}+</tool_call><|im_end|>+<|im_start|>user+<tool_response>+done+</tool_response><|im_end|>+<|im_start|>assistant+`,+ },+ {+ name: "user with multiple images",+ msgs: []api.Message{+ {Role: "user", Content: "Describe.", Images: []api.ImageData{api.ImageData("img1"), api.ImageData("img2")}},+ },+ expected: `<|im_start|>user+<|vision_start|><|image_pad|><|vision_end|><|vision_start|><|image_pad|><|vision_end|>Describe.<|im_end|>+<|im_start|>assistant+`,+ },+ {+ name: "user tool_response, no whitespace",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather?"},+ {+ Role: "assistant",+ Content: "I'll check.",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "get-current-weather", Arguments: map[string]any{"location": "Paris", "unit": "celsius"}}},+ },+ },+ {Role: "user", Content: "<tool_response>\n18\n</tool_response>"},+ {Role: "user", Content: "Thanks!"},+ },+ expected: `<|im_start|>user+What's the weather?<|im_end|>+<|im_start|>assistant+I'll check.+<tool_call>+{"name": "get-current-weather", "arguments": {"location": "Paris", "unit": "celsius"}}+</tool_call><|im_end|>+<|im_start|>user+<tool_response>+18+</tool_response><|im_end|>+<|im_start|>user+Thanks!<|im_end|>+<|im_start|>assistant+`,+ },+ {+ name: "user tool_response with surrounding whitespace",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather?"},+ {+ Role: "assistant",+ Content: "I'll check.",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "get-current-weather", Arguments: map[string]any{"location": "Paris", "unit": "celsius"}}},+ },+ },+ {Role: "user", Content: "\n\n\n\n<tool_response>\n18\n</tool_response> extra\n\n\n\n\n\n"},+ },+ expected: `<|im_start|>user+What's the weather?<|im_end|>+<|im_start|>assistant+I'll check.+<tool_call>+{"name": "get-current-weather", "arguments": {"location": "Paris", "unit": "celsius"}}+</tool_call><|im_end|>+<|im_start|>user+++++<tool_response>+18+</tool_response> extra++++++<|im_end|>+<|im_start|>assistant+`,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ rendered, err := (&Qwen3VLRenderer{false}).Render(tt.msgs, tt.tools, nil)+ if err != nil {+ t.Fatal(err)+ }+ if diff := cmp.Diff(rendered, tt.expected); diff != "" {+ t.Errorf("mismatch (-got +want):\n%s", diff)+ }+ })+ }+}diff --git a/model/renderers/qwen3vl_test.go b/model/renderers/qwen3vl_test.gonew file mode 100644index 000000000..6810a7c94--- /dev/null+++ b/model/renderers/qwen3vl_test.go@@ -0,0 +1,346 @@+package renderers++import (+ "testing"++ "github.com/google/go-cmp/cmp"+)++// TODO(drifkin): this will be moved to utils in the near future and used by other renderers as well+func TestMarshalWithSpaces(t *testing.T) {+ tests := []struct {+ name string+ input any+ expected string+ }{+ // basic formatting tests+ {+ name: "simple object",+ input: map[string]any{"key": "value"},+ expected: `{"key": "value"}`,+ },+ {+ name: "simple array",+ input: []any{"a", "b", "c"},+ expected: `["a", "b", "c"]`,+ },+ // escaped quotes+ {+ name: "escaped quote in string",+ input: map[string]any{"text": `quote"inside`},+ expected: `{"text": "quote\"inside"}`,+ },+ {+ name: "multiple escaped quotes",+ input: map[string]any{"text": `say "hello" and "goodbye"`},+ expected: `{"text": "say \"hello\" and \"goodbye\""}`,+ },+ // escaped backslashes+ {+ name: "escaped backslash",+ input: map[string]any{"path": `C:\windows\system32`},+ expected: `{"path": "C:\\windows\\system32"}`,+ },+ {+ name: "double backslash",+ input: map[string]any{"text": `test\\more`},+ expected: `{"text": "test\\\\more"}`,+ },+ {+ name: "backslash before quote",+ input: map[string]any{"text": `end with \"`},+ expected: `{"text": "end with \\\""}`,+ },+ // standard JSON escape sequences+ {+ name: "newline in string",+ input: map[string]any{"text": "line1\nline2"},+ expected: `{"text": "line1\nline2"}`,+ },+ {+ name: "tab in string",+ input: map[string]any{"text": "before\tafter"},+ expected: `{"text": "before\tafter"}`,+ },+ {+ name: "carriage return",+ input: map[string]any{"text": "before\rafter"},+ expected: `{"text": "before\rafter"}`,+ },+ {+ name: "multiple escape sequences",+ input: map[string]any{"text": "line1\nline2\ttab\rcarriage"},+ expected: `{"text": "line1\nline2\ttab\rcarriage"}`,+ },+ // strings containing colons and commas (no spaces should be added inside)+ {+ name: "colon in string",+ input: map[string]any{"url": "http://example.com"},+ expected: `{"url": "http://example.com"}`,+ },+ {+ name: "comma in string",+ input: map[string]any{"list": "apple, banana, cherry"},+ expected: `{"list": "apple, banana, cherry"}`,+ },+ {+ name: "colon and comma in string",+ input: map[string]any{"data": "key:value, key2:value2"},+ expected: `{"data": "key:value, key2:value2"}`,+ },+ // unicode characters+ {+ name: "emoji",+ input: map[string]any{"emoji": "😀🎉✨"},+ expected: `{"emoji": "😀🎉✨"}`,+ },+ {+ name: "chinese characters",+ input: map[string]any{"text": "你好世界"},+ expected: `{"text": "你好世界"}`,+ },+ {+ name: "arabic characters",+ input: map[string]any{"text": "مرحبا"},+ expected: `{"text": "مرحبا"}`,+ },+ {+ name: "mixed unicode and ascii",+ input: map[string]any{"text": "Hello 世界! 😀"},+ expected: `{"text": "Hello 世界! 😀"}`,+ },+ {+ name: "unicode with special symbols",+ input: map[string]any{"text": "®©™€£¥"},+ expected: `{"text": "®©™€£¥"}`,+ },+ // complex combinations - strings that look like JSON+ {+ name: "json string inside value",+ input: map[string]any{"nested": `{"key":"value"}`},+ expected: `{"nested": "{\"key\":\"value\"}"}`,+ },+ {+ name: "json array inside value",+ input: map[string]any{"array": `["a","b","c"]`},+ expected: `{"array": "[\"a\",\"b\",\"c\"]"}`,+ },+ // edge cases+ {+ name: "empty string",+ input: map[string]any{"empty": ""},+ expected: `{"empty": ""}`,+ },+ {+ name: "empty object",+ input: map[string]any{},+ expected: `{}`,+ },+ {+ name: "empty array",+ input: []any{},+ expected: `[]`,+ },+ {+ name: "numbers",+ input: map[string]any{"int": 42, "float": 3.14},+ expected: `{"float": 3.14, "int": 42}`,+ },+ {+ name: "boolean",+ input: map[string]any{"bool": true, "other": false},+ expected: `{"bool": true, "other": false}`,+ },+ {+ name: "null value",+ input: map[string]any{"value": nil},+ expected: `{"value": null}`,+ },+ // nested structures with complex strings+ {+ name: "nested object with escapes",+ input: map[string]any{+ "outer": map[string]any{+ "path": `C:\folder\file.txt`,+ "quote": `He said "hi"`,+ },+ },+ expected: `{"outer": {"path": "C:\\folder\\file.txt", "quote": "He said \"hi\""}}`,+ },+ {+ name: "array with unicode and escapes",+ input: []any{+ "normal",+ "with\nnewline",+ "with\"quote",+ "emoji😀",+ "colon:comma,",+ },+ expected: `["normal", "with\nnewline", "with\"quote", "emoji😀", "colon:comma,"]`,+ },+ {+ name: "backslash at positions before special chars",+ input: map[string]any{"text": `a\b:c\d,e`},+ expected: `{"text": "a\\b:c\\d,e"}`,+ },+ {+ name: "multiple backslashes before quote",+ input: map[string]any{"text": `ends\\"`},+ expected: `{"text": "ends\\\\\""}`,+ },+ {+ name: "unicode with escapes",+ input: map[string]any{"text": "Hello\n世界\t😀"},+ expected: `{"text": "Hello\n世界\t😀"}`,+ },++ // Real-world tool call example+ {+ name: "tool call arguments",+ input: map[string]any{+ "location": "San Francisco, CA",+ "unit": "fahrenheit",+ "format": "json",+ },+ expected: `{"format": "json", "location": "San Francisco, CA", "unit": "fahrenheit"}`,+ },+ {+ name: "complex tool arguments with escapes",+ input: map[string]any{+ "query": `SELECT * FROM "users" WHERE name = 'O'Brien'`,+ "description": "Fetch user\ndata from DB",+ "path": `C:\data\users.db`,+ },+ expected: `{"description": "Fetch user\ndata from DB", "path": "C:\\data\\users.db", "query": "SELECT * FROM \"users\" WHERE name = 'O'Brien'"}`,+ },+ {+ name: "unicode immediately adjacent to JSON structure chars",+ input: map[string]any{"😀key": "😀value", "test": "😀:😀,😀"},+ expected: `{"test": "😀:😀,😀", "😀key": "😀value"}`,+ },+ {+ name: "long unicode string stress test",+ input: map[string]any{"text": "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😔😕😖😗😘😙😚😛😜😝😞😟"},+ expected: `{"text": "😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😔😕😖😗😘😙😚😛😜😝😞😟"}`,+ },+ {+ name: "deeply nested with unicode everywhere",+ input: map[string]any{+ "😀": map[string]any{+ "你好": []any{"مرحبا", "®©™", "∑∫∂√"},+ },+ },+ expected: `{"😀": {"你好": ["مرحبا", "®©™", "∑∫∂√"]}}`,+ },+ {+ name: "unicode with all JSON special chars interleaved",+ input: map[string]any{"k😀:k": "v😀,v", "a:😀": "b,😀", "😀": ":,😀,:"},+ expected: `{"a:😀": "b,😀", "k😀:k": "v😀,v", "😀": ":,😀,:"}`,+ },+ {+ name: "combining diacritics and RTL text",+ input: map[string]any{"hebrew": "עִבְרִית", "combined": "é̀ñ", "mixed": "test:עִבְרִית,é̀ñ"},+ expected: `{"combined": "é̀ñ", "hebrew": "עִבְרִית", "mixed": "test:עִבְרִית,é̀ñ"}`,+ },+ {+ name: "pathological case: unicode + escapes + special chars",+ input: map[string]any{"😀": "test\n😀\"quote😀\\backslash😀:colon😀,comma😀"},+ expected: `{"😀": "test\n😀\"quote😀\\backslash😀:colon😀,comma😀"}`,+ },++ // all JSON structural characters inside strings+ {+ name: "braces and brackets in strings",+ input: map[string]any{"text": "test{with}braces[and]brackets"},+ expected: `{"text": "test{with}braces[and]brackets"}`,+ },+ {+ name: "braces and brackets with colons and commas",+ input: map[string]any{"code": "{key:value,[1,2,3]}"},+ expected: `{"code": "{key:value,[1,2,3]}"}`,+ },+ {+ name: "json-like string with all structural chars",+ input: map[string]any{"schema": `{"type":"object","properties":{"name":{"type":"string"},"items":{"type":"array"}}}`},+ expected: `{"schema": "{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\"},\"items\":{\"type\":\"array\"}}}"}`,+ },++ // forward slash tests (JSON allows \/ as an escape sequence)+ {+ name: "forward slash in URL",+ input: map[string]any{"url": "https://example.com/path/to/resource"},+ expected: `{"url": "https://example.com/path/to/resource"}`,+ },+ {+ name: "regex pattern with slashes",+ input: map[string]any{"regex": "/[a-z]+/gi"},+ expected: `{"regex": "/[a-z]+/gi"}`,+ },++ // all JSON escape sequences+ {+ name: "backspace escape",+ input: map[string]any{"text": "before\bafter"},+ expected: `{"text": "before\bafter"}`,+ },+ {+ name: "form feed escape",+ input: map[string]any{"text": "before\fafter"},+ expected: `{"text": "before\fafter"}`,+ },+ {+ name: "all standard escapes combined",+ input: map[string]any{"text": "\"\\\b\f\n\r\t"},+ expected: `{"text": "\"\\\b\f\n\r\t"}`,+ },++ // unicode escape sequences+ {+ name: "string that forces unicode escapes",+ input: map[string]any{"control": "\u0000\u0001\u001f"},+ expected: `{"control": "\u0000\u0001\u001f"}`,+ },++ // empty objects and arrays nested with strings+ {+ name: "nested empty structures with string values",+ input: map[string]any{"empty_obj": map[string]any{}, "empty_arr": []any{}, "text": "{}[]"},+ expected: `{"empty_arr": [], "empty_obj": {}, "text": "{}[]"}`,+ },++ // complex nesting with all structural characters+ {+ name: "deeply nested with all char types",+ input: map[string]any{+ "level1": map[string]any{+ "array": []any{+ map[string]any{"nested": "value:with,special{chars}[here]"},+ []any{"a", "b", "c"},+ },+ },+ },+ expected: `{"level1": {"array": [{"nested": "value:with,special{chars}[here]"}, ["a", "b", "c"]]}}`,+ },++ // string containing escaped structural characters+ {+ name: "string with multiple escape sequences and structural chars",+ input: map[string]any{"data": "test\"quote\"{brace}[bracket]:colon,comma\\backslash/slash"},+ expected: `{"data": "test\"quote\"{brace}[bracket]:colon,comma\\backslash/slash"}`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ result, err := marshalWithSpaces(tt.input)+ if err != nil {+ t.Fatalf("marshalWithSpaces failed: %v", err)+ }++ resultStr := string(result)+ if diff := cmp.Diff(resultStr, tt.expected); diff != "" {+ t.Errorf("mismatch (-got +want):\n%s", diff)+ }+ })+ }+}diff --git a/model/renderers/qwen3vl_thinking_test.go b/model/renderers/qwen3vl_thinking_test.gonew file mode 100644index 000000000..2bb2c2d0d--- /dev/null+++ b/model/renderers/qwen3vl_thinking_test.go@@ -0,0 +1,372 @@+package renderers++import (+ "testing"++ "github.com/google/go-cmp/cmp"+ "github.com/ollama/ollama/api"+)++func TestQwen3VLThinkingRenderer(t *testing.T) {+ tests := []struct {+ name string+ msgs []api.Message+ images []api.ImageData+ tools []api.Tool+ expected string+ }{+ {+ name: "basic",+ msgs: []api.Message{+ {Role: "system", Content: "You are a helpful assistant."},+ {Role: "user", Content: "Hello, how are you?"},+ },+ expected: `<|im_start|>system+You are a helpful assistant.<|im_end|>+<|im_start|>user+Hello, how are you?<|im_end|>+<|im_start|>assistant+<think>+`,+ },+ {+ name: "With thinking, end assistant.",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Content: "abc", Thinking: "To make this story interesting, I will speak in poetry."},+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+<think>+To make this story interesting, I will speak in poetry.+</think>++abc`,+ },+ {+ name: "With thinking, end assistant.",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Thinking: "To make this story interesting, I will speak in poetry."},+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+<think>+To make this story interesting, I will speak in poetry.`,+ },+ {+ name: "Multiple thinking",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Content: "abc", Thinking: "To make this story interesting, I will speak in poetry.<think>And I will speak in poetry after the first sentence.</think>"},+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+<think>+To make this story interesting, I will speak in poetry.<think>And I will speak in poetry after the first sentence.</think>+</think>++abc`, // NOTE: the second thinking tag is not captured+ },+ {+ name: "Multiple thinking, multiple messages.",+ msgs: []api.Message{+ {Role: "user", Content: "Tell me a story in two sentences."},+ {Role: "assistant", Thinking: "To make this story interesting, I will speak in poetry.", Content: "abc"},+ {Role: "user", Content: "What is the weather like in San Francisco?"},+ {Role: "assistant", Thinking: "Speak poetry after the first sentence.</think><think>Speak poetry after the second sentence."},+ },+ expected: `<|im_start|>user+Tell me a story in two sentences.<|im_end|>+<|im_start|>assistant+abc<|im_end|>+<|im_start|>user+What is the weather like in San Francisco?<|im_end|>+<|im_start|>assistant+<think>+Speak poetry after the first sentence.</think><think>Speak poetry after the second sentence.`,+ },+ // NOTE: Servers automatically prepend a [img-<n>] tag+ // {+ // name: "Image",+ // msgs: []api.Message{+ // {Role: "user", Content: "Describe this image.", Images: []api.ImageData{api.ImageData(IMAGE2_BASE64)}},+ // },+ // expected: `<|im_start|>user+ // [img-0]Describe this image.<|im_end|>+ // <|im_start|>assistant+ // <think>+ // `,+ // },++ // NOTE: Servers automatically prepend a [img-<n>] tag+ // {+ // name: "Multiple images",+ // msgs: []api.Message{+ // {Role: "user", Content: "Describe these images.", Images: []api.ImageData{api.ImageData(IMAGE1_BASE64), api.ImageData(IMAGE2_BASE64)}},+ // },+ // expected: `<|im_start|>user+ // [img-0][img-1]Describe these images.<|im_end|>+ // <|im_start|>assistant+ // <think>+ // `,+ // },++ // NOTE: solved with #12518: https://github.com/ollama/ollama/compare/main...drifkin/stable-tool-args+ // {+ // name: "with tools and response",+ // msgs: []api.Message{+ // {Role: "system", Content: "You are a helpful assistant with access to tools."},+ // {Role: "user", Content: "What's the weather like in New York?"},+ // {+ // Role: "assistant",+ // Content: "I'll check the weather in New York for you.",+ // ToolCalls: []api.ToolCall{+ // {+ // Function: api.ToolCallFunction{+ // Name: "get-current-weather",+ // Arguments: map[string]any{+ // "location": "New York",+ // "unit": "fahrenheit",+ // },+ // },+ // },+ // },+ // },+ // {Role: "tool", Content: "80", ToolName: "get-current-weather"},+ // {Role: "user", Content: "That sounds nice! What about San Francisco?"},+ // },+ // tools: []api.Tool{+ // {+ // Type: "function",+ // Function: api.ToolFunction{+ // Name: "get-current-weather",+ // Description: "Get the current weather for a location",+ // Parameters: api.ToolFunctionParameters{+ // Type: "object",+ // Required: []string{"location"},+ // Properties: map[string]api.ToolProperty{+ // "location": {+ // Type: api.PropertyType{"string"},+ // Description: "The city and state, e.g. San Francisco, CA",+ // },+ // "unit": {+ // Type: api.PropertyType{"string"},+ // Enum: []any{"celsius", "fahrenheit"},+ // Description: "The temperature unit",+ // },+ // },+ // },+ // },+ // },+ // },+ // expected: `<|im_start|>system+ // You are a helpful assistant with access to tools.++ // # Tools++ // You may call one or more functions to assist with the user query.++ // You are provided with function signatures within <tools></tools> XML tags:+ // <tools>+ // {"type": "function", "function": {"name": "get-current-weather", "description": "Get the current weather for a location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "The temperature unit"}}, "required": ["location"]}}}+ // </tools>++ // For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:+ // <tool_call>+ // {"name": <function-name>, "arguments": <args-json-object>}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // What's the weather like in New York?<|im_end|>+ // <|im_start|>assistant+ // I'll check the weather in New York for you.+ // <tool_call>+ // {"name": "get-current-weather", "arguments": {"location": "New York", "unit": "fahrenheit"}}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // <tool_response>+ // 80+ // </tool_response><|im_end|>+ // <|im_start|>user+ // That sounds nice! What about San Francisco?<|im_end|>+ // <|im_start|>assistant+ // <think>+ // `,+ // },++ // NOTE: solved with #12518: https://github.com/ollama/ollama/compare/main...drifkin/stable-tool-args+ // {+ // name: "With tools and response, multiple tool calls",+ // msgs: []api.Message{+ // {+ // Role: "system",+ // Content: "You are a helpful assistant with access to tools.",+ // },+ // {+ // Role: "user",+ // Content: "Call two tools for me: add and multiply.",+ // },+ // {+ // Role: "assistant",+ // Content: "Sure, I'll call both tools for you.",+ // ToolCalls: []api.ToolCall{+ // {+ // Function: api.ToolCallFunction{+ // Name: "add",+ // Arguments: map[string]any{+ // "a": 2,+ // "b": 3,+ // },+ // },+ // },+ // {+ // Function: api.ToolCallFunction{+ // Name: "multiply",+ // Arguments: map[string]any{+ // "x": 4,+ // "y": 5,+ // },+ // },+ // },+ // },+ // },+ // {+ // Role: "tool",+ // Content: "5",+ // ToolName: "add",+ // },+ // {+ // Role: "tool",+ // Content: "20",+ // ToolName: "multiply",+ // },+ // {+ // Role: "user",+ // Content: "Thanks! What are the results?",+ // },+ // },+ // tools: []api.Tool{+ // {+ // Type: "function",+ // Function: api.ToolFunction{+ // Name: "add",+ // Description: "Add two numbers",+ // Parameters: api.ToolFunctionParameters{+ // Type: "object",+ // Required: []string{"a", "b"},+ // Properties: map[string]api.ToolProperty{+ // "a": {Type: api.PropertyType{"integer"}, Description: "First number"},+ // "b": {Type: api.PropertyType{"integer"}, Description: "Second number"},+ // },+ // },+ // },+ // },+ // {+ // Type: "function",+ // Function: api.ToolFunction{+ // Name: "multiply",+ // Description: "Multiply two numbers",+ // Parameters: api.ToolFunctionParameters{+ // Type: "object",+ // Required: []string{"x", "y"},+ // Properties: map[string]api.ToolProperty{+ // "x": {Type: api.PropertyType{"integer"}, Description: "First factor"},+ // "y": {Type: api.PropertyType{"integer"}, Description: "Second factor"},+ // },+ // },+ // },+ // },+ // },+ // expected: `<|im_start|>system+ // You are a helpful assistant with access to tools.++ // # Tools++ // You may call one or more functions to assist with the user query.++ // You are provided with function signatures within <tools></tools> XML tags:+ // <tools>+ // {"type": "function", "function": {"name": "add", "description": "Add two numbers", "parameters": {"type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"]}}}+ // {"type": "function", "function": {"name": "multiply", "description": "Multiply two numbers", "parameters": {"type": "object", "properties": {"x": {"type": "integer"}, "y": {"type": "integer"}}, "required": ["x", "y"]}}}+ // </tools>++ // For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:+ // <tool_call>+ // {"name": <function-name>, "arguments": <args-json-object>}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // Call two tools for me: add and multiply.<|im_end|>+ // <|im_start|>assistant+ // Sure, I'll call both tools for you.+ // <tool_call>+ // {"name": "add", "arguments": {"a": 2, "b": 3}}+ // </tool_call>+ // <tool_call>+ // {"name": "multiply", "arguments": {"x": 4, "y": 5}}+ // </tool_call><|im_end|>+ // <|im_start|>user+ // <tool_response>+ // 5+ // </tool_response>+ // <tool_response>+ // 20+ // </tool_response><|im_end|>+ // <|im_start|>user+ // Thanks! What are the results?<|im_end|>+ // <|im_start|>assistant+ // <think>+ // `,+ // },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ rendered, err := (&Qwen3VLRenderer{true}).Render(tt.msgs, tt.tools, nil)+ if err != nil {+ t.Fatal(err)+ }+ if diff := cmp.Diff(rendered, tt.expected); diff != "" {+ t.Errorf("mismatch (-got +want):\n%s", diff)+ }+ })+ }+}++func TestFormatToolCallArgumentThinkingVL(t *testing.T) {+ tests := []struct {+ name string+ arg any+ expected string+ }{+ {+ name: "string",+ arg: "foo",+ expected: "foo",+ },+ {+ name: "map",+ arg: map[string]any{"foo": "bar"},+ expected: "{\"foo\":\"bar\"}",+ },+ {+ name: "number",+ arg: 1,+ expected: "1",+ },+ {+ name: "boolean",+ arg: true,+ expected: "true",+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := formatToolCallArgument(tt.arg)+ if got != tt.expected {+ t.Errorf("formatToolCallArgument(%v) = %v, want %v", tt.arg, got, tt.expected)+ }+ })+ }+}diff --git a/model/renderers/renderer.go b/model/renderers/renderer.goindex 2dfb51e49..01e0c6ee2 100644--- a/model/renderers/renderer.go+++ b/model/renderers/renderer.go@@ -1,25 +1,19 @@package renderers-import (- "fmt"+import "github.com/ollama/ollama/api"- "github.com/ollama/ollama/api"-)--type rendererFunc func([]api.Message, []api.Tool, *api.ThinkValue) (string, error)--func RenderWithRenderer(name string, msgs []api.Message, tools []api.Tool, think *api.ThinkValue) (string, error) {- renderer := rendererForName(name)- if renderer == nil {- return "", fmt.Errorf("unknown renderer %q", name)- }- return renderer(msgs, tools, think)+type Renderer interface {+ Render(messages []api.Message, tools []api.Tool, think *api.ThinkValue) (string, error)}-func rendererForName(name string) rendererFunc {+func RendererForName(name string) Renderer {switch name {case "qwen3-coder":- return Qwen3CoderRenderer+ renderer := &Qwen3CoderRenderer{}+ return renderer+ case "qwen3-vl-instruct":+ renderer := &Qwen3VLRenderer{false}+ return rendererdefault:return nil}diff --git a/openai/openai.go b/openai/openai.goindex a01c5d35c..0e2f7313a 100644--- a/openai/openai.go+++ b/openai/openai.go@@ -456,6 +456,11 @@ func FromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {types := []string{"jpeg", "jpg", "png", "webp"}valid := false+ // support blank mime type to match api/chat taking just unadorned base64+ if strings.HasPrefix(url, "data:;base64,") {+ url = strings.TrimPrefix(url, "data:;base64,")+ valid = true+ }for _, t := range types {prefix := "data:image/" + t + ";base64,"if strings.HasPrefix(url, prefix) {diff --git a/server/prompt.go b/server/prompt.goindex 217591982..f1f7cec4d 100644--- a/server/prompt.go+++ b/server/prompt.go@@ -106,7 +106,8 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.func renderPrompt(m *Model, msgs []api.Message, tools []api.Tool, think *api.ThinkValue) (string, error) {if m.Config.Renderer != "" {- rendered, err := renderers.RenderWithRenderer(m.Config.Renderer, msgs, tools, think)+ renderer := renderers.RendererForName(m.Config.Renderer)+ rendered, err := renderer.Render(msgs, tools, think)if err != nil {return "", err}