Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a <think> block after a line end at a Chat reply's head is thinking (jenryai on X)

yetonefc16214dac06
diff --git a/internal/gateway/chat.go b/internal/gateway/chat.go
index 0fca9b3ef..12f1f9666 100644
--- a/internal/gateway/chat.go
+++ b/internal/gateway/chat.go
@@ -1095,6 +1095,11 @@ type chatDecoder struct {
trim bool
}
+// leadSpace is the white space a leading block of thinking may come
+// after: a relay writing Windows line ends puts "\r\n" before its <think>
+// (grok2api_python's grok-3-search: "\r\n<think>…</think>\r\n").
+const leadSpace = " \t\r\n"
+
// thoughtTags are the tags a leading block of thinking comes between in a
// Chat reply's text, by its opening tag.
var thoughtTags = map[string]string{"<thought>": "</thought>", "<think>": "</think>"}
@@ -1104,7 +1109,7 @@ var thoughtTags = map[string]string{"<thought>": "</thought>", "<think>": "</thi
func (d *chatDecoder) text(s string, emit func(Event)) {
if !d.past && !d.thought {
d.lead += s
- lead := strings.TrimLeft(d.lead, " \n")
+ lead := strings.TrimLeft(d.lead, leadSpace)
for open := range thoughtTags {
if len(lead) < len(open) && strings.HasPrefix(open, lead) {
return
@@ -1145,7 +1150,7 @@ func (d *chatDecoder) text(s string, emit func(Event)) {
}
}
if d.trim {
- if s = strings.TrimLeft(s, "\n"); s != "" {
+ if s = strings.TrimLeft(s, "\r\n"); s != "" {
d.trim = false
}
}
@@ -1218,7 +1223,7 @@ func (m *mdCode) find(s, close string, last bool) (at, hold int) {
// afterThought is t without a leading <think>…</think> block, a whole
// reply's thinking where its API leaves it in the text; t when it has none.
func afterThought(t string) string {
- rest, ok := strings.CutPrefix(t, "<think>")
+ rest, ok := strings.CutPrefix(strings.TrimLeft(t, leadSpace), "<think>")
if !ok {
return t
}
@@ -1241,7 +1246,7 @@ func (d *chatDecoder) markedThought(s string, emit func(Event)) {
return
}
tagged := d.thought || d.lead != ""
- if lead := strings.TrimLeft(s, " \n"); !tagged && !d.past {
+ if lead := strings.TrimLeft(s, leadSpace); !tagged && !d.past {
tagged = lead == ""
for open := range thoughtTags {
tagged = tagged || strings.HasPrefix(lead, open) || strings.HasPrefix(open, lead)
diff --git a/internal/gateway/grok_relay_think_test.go b/internal/gateway/grok_relay_think_test.go
new file mode 100644
index 000000000..e463ec68e
--- /dev/null
+++ b/internal/gateway/grok_relay_think_test.go
@@ -0,0 +1,98 @@
+package gateway
+
+import (
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// grokRelaySearchStream is a reply as grok2api_python (xLmiler's Grok
+// reverse proxy) streams its grok-3-search model with search results
+// shown: the results come first, as json.dumps writes them, between
+// <think> tags led by a Windows line end ("\r\n<think>{results}</think>\r\n"
+// in process_model_response), then Grok's answer token by token, with no
+// finish_reason before [DONE].
+const grokRelaySearchStream = `data: {"id": "chatcmpl-6c1f0e3a-1b7e-4a51-9a51-0c6e8f1f2d11", "created": 1761700000, "model": "grok-3-search", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "\r\n<think>\r\n<details><summary>\u8d44\u6599[0]: magpie</summary>\r\nA local LLM gateway\r\n\n[Link](https://usemagpie.ai)\r\n</details></think>\r\n"}}]}
+
+data: {"id": "chatcmpl-1f0b2c4d-8e9a-4b3c-a2d1-5e6f7a8b9c0d", "created": 1761700000, "model": "grok-3-search", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "Magpie"}}]}
+
+data: {"id": "chatcmpl-2a3b4c5d-6e7f-4a8b-9c0d-1e2f3a4b5c6d", "created": 1761700000, "model": "grok-3-search", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": " is a gateway."}}]}
+
+data: [DONE]
+
+`
+
+// A Grok relay that puts a line end before its leading <think> block
+// (jenryai on X: the thinking came out as the answer's text) has that
+// block taken as thinking on every route a translated client takes,
+// streamed to it or not, and the answer starts at its first word.
+func TestGrokRelayThinkAfterLineEnd(t *testing.T) {
+ for _, c := range []struct {
+ name, path, body string
+ }{
+ {"messages", "/v1/messages", `{"model":"grokr/grok-3-search","stream":true,"max_tokens":100,"messages":[{"role":"user","content":"magpie?"}]}`},
+ {"responses", "/v1/responses", `{"model":"grokr/grok-3-search","stream":true,"input":[{"role":"user","content":"magpie?"}]}`},
+ {"messages whole", "/v1/messages", `{"model":"grokr/grok-3-search","max_tokens":100,"messages":[{"role":"user","content":"magpie?"}]}`},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ t.Setenv("XDG_CONFIG_HOME", t.TempDir())
+ t.Setenv("XDG_CACHE_HOME", t.TempDir())
+ up := httptest.NewServer(&fake{t: t, reply: grokRelaySearchStream})
+ t.Cleanup(up.Close)
+ if err := provider.Save(provider.Provider{ID: "grokr", Name: "Grok relay", Key: "k", Models: []string{"grok-3-search"}, Chat: up.URL + "/v1"}); err != nil {
+ t.Fatal(err)
+ }
+ code, out := post(t, c.path, c.body)
+ if code != 200 {
+ t.Fatalf("code %d: %s", code, out)
+ }
+ think, text := grokRelayParts(t, out)
+ if !strings.Contains(think, "资料[0]: magpie") || !strings.Contains(think, "[Link](https://usemagpie.ai)") {
+ t.Errorf("thinking %q, want the search results", think)
+ }
+ if text != "Magpie is a gateway." {
+ t.Errorf("answer %q, want %q", text, "Magpie is a gateway.")
+ }
+ })
+ }
+}
+
+// grokRelayParts is the thinking and the answer's text in a Messages or
+// Responses reply, streamed or whole.
+func grokRelayParts(t *testing.T, out string) (think, text string) {
+ t.Helper()
+ if strings.HasPrefix(strings.TrimSpace(out), "{") {
+ for _, ev := range events("data: " + out + "\n\n") {
+ blocks, _ := ev["content"].([]any)
+ for _, b := range blocks {
+ m, _ := b.(map[string]any)
+ if v, ok := m["thinking"].(string); ok {
+ think += v
+ }
+ if v, ok := m["text"].(string); ok {
+ text += v
+ }
+ }
+ }
+ return
+ }
+ for _, ev := range events(out) {
+ switch ev["type"] {
+ case "content_block_delta":
+ d, _ := ev["delta"].(map[string]any)
+ if v, ok := d["thinking"].(string); ok {
+ think += v
+ }
+ if v, ok := d["text"].(string); ok {
+ text += v
+ }
+ case "response.reasoning_summary_text.delta", "response.reasoning_text.delta":
+ think += ev["delta"].(string)
+ case "response.output_text.delta":
+ text += ev["delta"].(string)
+ }
+ }
+ return
+}
diff --git a/internal/gateway/minimax_think_test.go b/internal/gateway/minimax_think_test.go
index e1eacc50a..83c1e9ec3 100644
--- a/internal/gateway/minimax_think_test.go
+++ b/internal/gateway/minimax_think_test.go
@@ -137,6 +137,8 @@ func TestChatThinkTags(t *testing.T) {
{"a thought's close doesn't end it", []string{"<think>a</thought>b</think>c"}, "a</thought>b", "c"},
{"a think tag later", []string{"Hi <think>no</think>"}, "", "Hi <think>no</think>"},
{"<thin is text", []string{"<thin", "g>"}, "", "<thing>"},
+ {"after a Windows line end", []string{"\r", "\n<think>a</think>\r\n", "b"}, "a", "b"},
+ {"after a tab", []string{"\t<think>a</think>\n\nb"}, "a", "b"},
} {
t.Run(c.name, func(t *testing.T) {
if think, text := run(c.parts...); think != c.think || text != c.text {
diff --git a/internal/gateway/think_code_test.go b/internal/gateway/think_code_test.go
index e5ea65ad6..348787d73 100644
--- a/internal/gateway/think_code_test.go
+++ b/internal/gateway/think_code_test.go
@@ -209,4 +209,7 @@ func TestAfterThoughtCloseInCode(t *testing.T) {
if got := afterThought("<think>never closed"); got != "<think>never closed" {
t.Errorf("afterThought of an open block = %q", got)
}
+ if got := afterThought("\r\n<think>a</think>\r\nb"); got != "b" {
+ t.Errorf("afterThought after a line end = %q", got)
+ }
}