Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(openaicompletions): keep streamed tool calls without an index apart (#4593)
alexcastgonz93ec8dad0ec2
diff --git a/internal/models/api/openaicompletions/client.go b/internal/models/api/openaicompletions/client.goindex d001a97f2..2c045f6d4 100644--- a/internal/models/api/openaicompletions/client.go+++ b/internal/models/api/openaicompletions/client.go@@ -191,7 +191,47 @@ func (c *Client) reasoningText(m rawMessage) string {return ""}-func (c *Client) decodeToolCalls(raws []json.RawMessage) ([]types.LLMToolCall, []api.ToolCallDelta, error) {+// streamToolIndex gives index-less streamed tool calls a stable index.+//+// OpenAI numbers the calls of a response with "index". Some compatible servers+// (Gemini's OpenAI endpoint among them) leave it out and send each parallel+// call whole in its own chunk, which would put every call at the position it+// has inside its chunk, index 0, and merge them into one. Without an index a+// fragment that carries an id different from the previous call's starts a new+// call; one without an id, or repeating the id, continues the previous call.+type streamToolIndex struct {+ started bool+ lastID string+ last int+ next int+}++func (s *streamToolIndex) resolve(explicit *int, id string) int {+ if explicit != nil {+ s.started, s.last = true, *explicit+ if id != "" {+ s.lastID = id+ }+ s.next = max(s.next, *explicit+1)+ return *explicit+ }+ if !s.started || (id != "" && id != s.lastID) {+ s.started = true+ s.last = s.next+ s.next+++ }+ if id != "" {+ s.lastID = id+ }+ return s.last+}++// decodeToolCalls decodes the tool calls of a message or of a streamed delta.+// For a stream, pass the stream's streamToolIndex to number index-less calls+// across chunks; without it a missing index falls back to the position in raws.+func (c *Client) decodeToolCalls(+ raws []json.RawMessage, indexes *streamToolIndex,+) ([]types.LLMToolCall, []api.ToolCallDelta, error) {calls := make([]types.LLMToolCall, 0, len(raws))deltas := make([]api.ToolCallDelta, 0, len(raws))for i, raw := range raws {@@ -203,7 +243,9 @@ func (c *Client) decodeToolCalls(raws []json.RawMessage) ([]types.LLMToolCall, [return nil, nil, fmt.Errorf("decode tool call %d: %w", i, err)}idx := i- if tc.Index != nil {+ if indexes != nil {+ idx = indexes.resolve(tc.Index, tc.ID)+ } else if tc.Index != nil {idx = *tc.Index}call := types.LLMToolCall{@@ -265,7 +307,7 @@ func (c *Client) parseResponse(raw []byte) (*types.ChatResponse, error) {result.ReasoningMetadata = types.ProviderMetadata{metadataReasoningDetails: choice.Message.ReasoningDetails}}if len(choice.Message.ToolCalls) > 0 {- calls, _, err := c.decodeToolCalls(choice.Message.ToolCalls)+ calls, _, err := c.decodeToolCalls(choice.Message.ToolCalls, nil)if err != nil {return nil, err}@@ -304,6 +346,7 @@ func (c *Client) processStream(assembler := api.NewStreamAssembler(ctx, c.cfg.Endpoint.Model)reader := api.NewSSEReader(body)var reasoningDetails []json.RawMessage+ toolIndexes := &streamToolIndex{}finish := func(end func(chan<- types.StreamResponse)) {if len(reasoningDetails) > 0 {@@ -377,7 +420,7 @@ func (c *Client) processStream(}}if len(choice.Delta.ToolCalls) > 0 {- _, deltas, err := c.decodeToolCalls(choice.Delta.ToolCalls)+ _, deltas, err := c.decodeToolCalls(choice.Delta.ToolCalls, toolIndexes)if err != nil {assembler.Fail(ch, err)returndiff --git a/internal/models/api/openaicompletions/stream_tool_index_test.go b/internal/models/api/openaicompletions/stream_tool_index_test.gonew file mode 100644index 000000000..449105f50--- /dev/null+++ b/internal/models/api/openaicompletions/stream_tool_index_test.go@@ -0,0 +1,111 @@+package openaicompletions++import (+ "context"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// streamCalls runs the given SSE data lines through ChatStream and returns the+// tool calls of the final chunk.+func streamCalls(t *testing.T, lines ...string) []types.LLMToolCall {+ t.Helper()+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "text/event-stream")+ for _, l := range lines {+ _, _ = w.Write([]byte("data: " + l + "\n\n"))+ }+ _, _ = w.Write([]byte("data: [DONE]\n\n"))+ }))+ defer server.Close()++ c := New(Config{+ Endpoint: api.Endpoint{BaseURL: server.URL, Model: "m"}, Settings: api.DefaultOpenAICompletions(),+ })+ ch, err := c.ChatStream(context.Background(), []api.Message{{Role: "user", Content: "hi"}}, nil)+ require.NoError(t, err)++ var last types.StreamResponse+ for chunk := range ch {+ require.NotEqual(t, types.ResponseTypeError, chunk.ResponseType, chunk.Content)+ if chunk.Done {+ last = chunk+ }+ }+ return last.ToolCalls+}++func toolChunk(calls string, finish string) string {+ f := "null"+ if finish != "" {+ f = `"` + finish + `"`+ }+ return `{"choices":[{"delta":{"tool_calls":[` + calls + `]},"finish_reason":` + f + `}]}`+}++func TestChatStream_IndexedToolCallsStayApart(t *testing.T) {+ calls := streamCalls(t,+ toolChunk(`{"index":0,"id":"a","type":"function","function":{"name":"get_weather","arguments":""}}`, ""),+ toolChunk(`{"index":0,"function":{"arguments":"{\"city\":\"Paris\"}"}}`, ""),+ toolChunk(+ `{"index":1,"id":"b","type":"function","function":{"name":"get_time","arguments":"{}"}}`,+ "tool_calls"),+ )+ require.Len(t, calls, 2)+ assert.Equal(t, "get_weather", calls[0].Function.Name)+ assert.Equal(t, `{"city":"Paris"}`, calls[0].Function.Arguments)+ assert.Equal(t, "get_time", calls[1].Function.Name)+}++func TestChatStream_ParallelCallsWithoutIndexStayApart(t *testing.T) {+ // Each call arrives whole in its own chunk and no chunk has an index.+ calls := streamCalls(t,+ toolChunk(+ `{"id":"a","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}`,+ ""),+ toolChunk(+ `{"id":"b","type":"function","function":{"name":"get_time","arguments":"{\"tz\":\"UTC\"}"}}`,+ "tool_calls"),+ )+ require.Len(t, calls, 2)+ assert.Equal(t, "a", calls[0].ID)+ assert.Equal(t, "get_weather", calls[0].Function.Name)+ assert.Equal(t, `{"city":"Paris"}`, calls[0].Function.Arguments)+ assert.Equal(t, "b", calls[1].ID)+ assert.Equal(t, "get_time", calls[1].Function.Name)+ assert.Equal(t, `{"tz":"UTC"}`, calls[1].Function.Arguments)+}++func TestChatStream_TwoCallsInOneChunkWithoutIndexStayApart(t *testing.T) {+ calls := streamCalls(t,+ toolChunk(+ `{"id":"a","type":"function","function":{"name":"get_weather","arguments":"{}"}},`++ `{"id":"b","type":"function","function":{"name":"get_time","arguments":"{}"}}`,+ "tool_calls"),+ )+ require.Len(t, calls, 2)+ assert.Equal(t, "get_weather", calls[0].Function.Name)+ assert.Equal(t, "get_time", calls[1].Function.Name)+}++func TestChatStream_FragmentsWithoutIndexContinueTheCall(t *testing.T) {+ // Name and id first, then argument fragments with neither, or with the+ // id repeated.+ calls := streamCalls(t,+ toolChunk(`{"id":"a","type":"function","function":{"name":"get_weather","arguments":""}}`, ""),+ toolChunk(`{"function":{"arguments":"{\"city\":"}}`, ""),+ toolChunk(`{"id":"a","function":{"arguments":"\"Paris\"}"}}`, "tool_calls"),+ )+ require.Len(t, calls, 1)+ assert.Equal(t, "get_weather", calls[0].Function.Name)+ assert.Equal(t, `{"city":"Paris"}`, calls[0].Function.Arguments)+ assert.True(t, strings.HasPrefix(calls[0].ID, "a"))+}