Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(openaicompletions): keep streamed tool calls without an index apart (#4593)

alexcastgonz93ec8dad0ec2
diff --git a/internal/models/api/openaicompletions/client.go b/internal/models/api/openaicompletions/client.go
index d001a97f2..2c045f6d4 100644
--- a/internal/models/api/openaicompletions/client.go
+++ b/internal/models/api/openaicompletions/client.go
@@ -191,7 +191,47 @@ func (c *Client) reasoningText(m rawMessage) string {
return ""
}
-func (c *Client) decodeToolCalls(raws []json.RawMessage) ([]types.LLMToolCall, []api.ToolCallDelta, error) {
+// streamToolIndex gives index-less streamed tool calls a stable index.
+//
+// OpenAI numbers the calls of a response with "index". Some compatible servers
+// (Gemini's OpenAI endpoint among them) leave it out and send each parallel
+// call whole in its own chunk, which would put every call at the position it
+// has inside its chunk, index 0, and merge them into one. Without an index a
+// fragment that carries an id different from the previous call's starts a new
+// call; one without an id, or repeating the id, continues the previous call.
+type streamToolIndex struct {
+ started bool
+ lastID string
+ last int
+ next int
+}
+
+func (s *streamToolIndex) resolve(explicit *int, id string) int {
+ if explicit != nil {
+ s.started, s.last = true, *explicit
+ if id != "" {
+ s.lastID = id
+ }
+ s.next = max(s.next, *explicit+1)
+ return *explicit
+ }
+ if !s.started || (id != "" && id != s.lastID) {
+ s.started = true
+ s.last = s.next
+ s.next++
+ }
+ if id != "" {
+ s.lastID = id
+ }
+ return s.last
+}
+
+// decodeToolCalls decodes the tool calls of a message or of a streamed delta.
+// For a stream, pass the stream's streamToolIndex to number index-less calls
+// across chunks; without it a missing index falls back to the position in raws.
+func (c *Client) decodeToolCalls(
+ raws []json.RawMessage, indexes *streamToolIndex,
+) ([]types.LLMToolCall, []api.ToolCallDelta, error) {
calls := make([]types.LLMToolCall, 0, len(raws))
deltas := make([]api.ToolCallDelta, 0, len(raws))
for i, raw := range raws {
@@ -203,7 +243,9 @@ func (c *Client) decodeToolCalls(raws []json.RawMessage) ([]types.LLMToolCall, [
return nil, nil, fmt.Errorf("decode tool call %d: %w", i, err)
}
idx := i
- if tc.Index != nil {
+ if indexes != nil {
+ idx = indexes.resolve(tc.Index, tc.ID)
+ } else if tc.Index != nil {
idx = *tc.Index
}
call := types.LLMToolCall{
@@ -265,7 +307,7 @@ func (c *Client) parseResponse(raw []byte) (*types.ChatResponse, error) {
result.ReasoningMetadata = types.ProviderMetadata{metadataReasoningDetails: choice.Message.ReasoningDetails}
}
if len(choice.Message.ToolCalls) > 0 {
- calls, _, err := c.decodeToolCalls(choice.Message.ToolCalls)
+ calls, _, err := c.decodeToolCalls(choice.Message.ToolCalls, nil)
if err != nil {
return nil, err
}
@@ -304,6 +346,7 @@ func (c *Client) processStream(
assembler := api.NewStreamAssembler(ctx, c.cfg.Endpoint.Model)
reader := api.NewSSEReader(body)
var reasoningDetails []json.RawMessage
+ toolIndexes := &streamToolIndex{}
finish := func(end func(chan<- types.StreamResponse)) {
if len(reasoningDetails) > 0 {
@@ -377,7 +420,7 @@ func (c *Client) processStream(
}
}
if len(choice.Delta.ToolCalls) > 0 {
- _, deltas, err := c.decodeToolCalls(choice.Delta.ToolCalls)
+ _, deltas, err := c.decodeToolCalls(choice.Delta.ToolCalls, toolIndexes)
if err != nil {
assembler.Fail(ch, err)
return
diff --git a/internal/models/api/openaicompletions/stream_tool_index_test.go b/internal/models/api/openaicompletions/stream_tool_index_test.go
new file mode 100644
index 000000000..449105f50
--- /dev/null
+++ b/internal/models/api/openaicompletions/stream_tool_index_test.go
@@ -0,0 +1,111 @@
+package openaicompletions
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// streamCalls runs the given SSE data lines through ChatStream and returns the
+// tool calls of the final chunk.
+func streamCalls(t *testing.T, lines ...string) []types.LLMToolCall {
+ t.Helper()
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ for _, l := range lines {
+ _, _ = w.Write([]byte("data: " + l + "\n\n"))
+ }
+ _, _ = w.Write([]byte("data: [DONE]\n\n"))
+ }))
+ defer server.Close()
+
+ c := New(Config{
+ Endpoint: api.Endpoint{BaseURL: server.URL, Model: "m"}, Settings: api.DefaultOpenAICompletions(),
+ })
+ ch, err := c.ChatStream(context.Background(), []api.Message{{Role: "user", Content: "hi"}}, nil)
+ require.NoError(t, err)
+
+ var last types.StreamResponse
+ for chunk := range ch {
+ require.NotEqual(t, types.ResponseTypeError, chunk.ResponseType, chunk.Content)
+ if chunk.Done {
+ last = chunk
+ }
+ }
+ return last.ToolCalls
+}
+
+func toolChunk(calls string, finish string) string {
+ f := "null"
+ if finish != "" {
+ f = `"` + finish + `"`
+ }
+ return `{"choices":[{"delta":{"tool_calls":[` + calls + `]},"finish_reason":` + f + `}]}`
+}
+
+func TestChatStream_IndexedToolCallsStayApart(t *testing.T) {
+ calls := streamCalls(t,
+ toolChunk(`{"index":0,"id":"a","type":"function","function":{"name":"get_weather","arguments":""}}`, ""),
+ toolChunk(`{"index":0,"function":{"arguments":"{\"city\":\"Paris\"}"}}`, ""),
+ toolChunk(
+ `{"index":1,"id":"b","type":"function","function":{"name":"get_time","arguments":"{}"}}`,
+ "tool_calls"),
+ )
+ require.Len(t, calls, 2)
+ assert.Equal(t, "get_weather", calls[0].Function.Name)
+ assert.Equal(t, `{"city":"Paris"}`, calls[0].Function.Arguments)
+ assert.Equal(t, "get_time", calls[1].Function.Name)
+}
+
+func TestChatStream_ParallelCallsWithoutIndexStayApart(t *testing.T) {
+ // Each call arrives whole in its own chunk and no chunk has an index.
+ calls := streamCalls(t,
+ toolChunk(
+ `{"id":"a","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}`,
+ ""),
+ toolChunk(
+ `{"id":"b","type":"function","function":{"name":"get_time","arguments":"{\"tz\":\"UTC\"}"}}`,
+ "tool_calls"),
+ )
+ require.Len(t, calls, 2)
+ assert.Equal(t, "a", calls[0].ID)
+ assert.Equal(t, "get_weather", calls[0].Function.Name)
+ assert.Equal(t, `{"city":"Paris"}`, calls[0].Function.Arguments)
+ assert.Equal(t, "b", calls[1].ID)
+ assert.Equal(t, "get_time", calls[1].Function.Name)
+ assert.Equal(t, `{"tz":"UTC"}`, calls[1].Function.Arguments)
+}
+
+func TestChatStream_TwoCallsInOneChunkWithoutIndexStayApart(t *testing.T) {
+ calls := streamCalls(t,
+ toolChunk(
+ `{"id":"a","type":"function","function":{"name":"get_weather","arguments":"{}"}},`+
+ `{"id":"b","type":"function","function":{"name":"get_time","arguments":"{}"}}`,
+ "tool_calls"),
+ )
+ require.Len(t, calls, 2)
+ assert.Equal(t, "get_weather", calls[0].Function.Name)
+ assert.Equal(t, "get_time", calls[1].Function.Name)
+}
+
+func TestChatStream_FragmentsWithoutIndexContinueTheCall(t *testing.T) {
+ // Name and id first, then argument fragments with neither, or with the
+ // id repeated.
+ calls := streamCalls(t,
+ toolChunk(`{"id":"a","type":"function","function":{"name":"get_weather","arguments":""}}`, ""),
+ toolChunk(`{"function":{"arguments":"{\"city\":"}}`, ""),
+ toolChunk(`{"id":"a","function":{"arguments":"\"Paris\"}"}}`, "tool_calls"),
+ )
+ require.Len(t, calls, 1)
+ assert.Equal(t, "get_weather", calls[0].Function.Name)
+ assert.Equal(t, `{"city":"Paris"}`, calls[0].Function.Arguments)
+ assert.True(t, strings.HasPrefix(calls[0].ID, "a"))
+}