Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: chat did not pass model config to provider (#18485)

Haruko38686ecdd85887d
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.go
index 2662adb607..8e29359328 100644
--- a/internal/entity/models/base_model.go
+++ b/internal/entity/models/base_model.go
@@ -517,6 +517,10 @@ func buildRequestBody(cfg *ChatConfig, modelName string, messages []Message, str
reqBody["top_p"] = *cfg.TopP
}
+ if cfg.MaxTokens != nil {
+ reqBody["max_tokens"] = *cfg.MaxTokens
+ }
+
if cfg.Stop != nil {
reqBody["stop"] = *cfg.Stop
}
diff --git a/internal/entity/models/xiaomi_test.go b/internal/entity/models/xiaomi_test.go
index c8162e9b4d..8e795cb4be 100644
--- a/internal/entity/models/xiaomi_test.go
+++ b/internal/entity/models/xiaomi_test.go
@@ -88,9 +88,6 @@ func TestXiaomiChatHappyPath(t *testing.T) {
if body["stream"] != false {
t.Errorf("stream=%v want false", body["stream"])
}
- if body["max_tokens"] != nil {
- t.Errorf("max_tokens must not be sent: %v", body["max_tokens"])
- }
if body["max_completion_tokens"] != float64(1024) {
t.Errorf("max_completion_tokens=%v", body["max_completion_tokens"])
}
diff --git a/internal/handler/chat_session.go b/internal/handler/chat_session.go
index a75a1bc509..d6c48aca66 100644
--- a/internal/handler/chat_session.go
+++ b/internal/handler/chat_session.go
@@ -205,6 +205,10 @@ func (h *ChatSessionHandler) ChatCompletions(c *gin.Context) {
genConfig["presence_penalty"] = *req.PresencePenalty
}
if req.MaxTokens != nil {
+ if *req.MaxTokens <= 0 {
+ common.ErrorWithCode(c, common.CodeBadRequest, "`max_tokens` must be greater than 0.")
+ return
+ }
genConfig["max_tokens"] = *req.MaxTokens
}
@@ -237,7 +241,7 @@ func (h *ChatSessionHandler) ChatCompletions(c *gin.Context) {
"llm_id",
"pass_all_history_messages", "pass_all_history",
"store_history_messages", "store_history",
- "legacy", "stream",
+ "legacy", "stream", "thinking",
"temperature", "top_p", "frequency_penalty", "presence_penalty", "max_tokens",
}
for _, key := range knownKeys {
diff --git a/internal/handler/chat_session_test.go b/internal/handler/chat_session_test.go
index 4913703b04..c27db504b8 100644
--- a/internal/handler/chat_session_test.go
+++ b/internal/handler/chat_session_test.go
@@ -74,3 +74,30 @@ func TestChatSessionHandlerUpdateMessageFeedback_RejectsEmptyJSONObject(t *testi
t.Fatalf("message=%v", got)
}
}
+
+func TestChatSessionHandlerChatCompletionsRejectsNonPositiveMaxTokens(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ recorder := httptest.NewRecorder()
+ ctx, _ := gin.CreateTestContext(recorder)
+ ctx.Request = httptest.NewRequest(http.MethodPost, "/api/v1/chat/completions", strings.NewReader(`{
+ "messages":[{"role":"user","content":"hi"}],
+ "max_tokens":0
+ }`))
+ ctx.Request.Header.Set("Content-Type", "application/json")
+ ctx.Set("user", &entity.User{ID: "user-1"})
+
+ handler := NewChatSessionHandler(service.NewChatSessionService(), nil)
+ handler.ChatCompletions(ctx)
+
+ var body map[string]interface{}
+ if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
+ t.Fatalf("decode response body: %v", err)
+ }
+ if got := body["code"]; got != float64(common.CodeBadRequest) {
+ t.Fatalf("code=%v", got)
+ }
+ if got := body["message"]; got != "`max_tokens` must be greater than 0." {
+ t.Fatalf("message=%v", got)
+ }
+}
diff --git a/internal/handler/openai_chat_test.go b/internal/handler/openai_chat_test.go
index 2e4e6177b7..7cfe72fce7 100644
--- a/internal/handler/openai_chat_test.go
+++ b/internal/handler/openai_chat_test.go
@@ -203,6 +203,22 @@ func TestChatCompletions_RejectsInvalidJSON(t *testing.T) {
}
}
+func TestChatCompletions_RejectsNonPositiveMaxTokens(t *testing.T) {
+ h := NewOpenAIChatHandler(service.NewOpenAIChatService())
+ c, w := newOpenAITestContext(t, "c1", `{
+ "model": "model",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 0
+ }`)
+
+ h.OpenAIChatCompletions(c)
+
+ respBody := w.Body.String()
+ if !strings.Contains(respBody, "`max_tokens` must be greater than 0.") {
+ t.Fatalf("expected max_tokens validation error, got %s", respBody)
+ }
+}
+
// TestChatCompletions_SilentlyDropsTopLevelStop verifies that top-level
// `stop` is silently dropped rather than rejected. The field is not declared
// on OpenAIChatRequest, so Go's json.Unmarshal discards it — matching the
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 7c6d198cf3..ad8be35d60 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -190,7 +190,7 @@ func (s *ChatPipelineService) AsyncChat(
}
if !hasKBs && !useWebSearch {
- return s.AsyncChatSolo(ctx, userID, chat, messages, stream)
+ return s.AsyncChatSolo(ctx, userID, chat, messages, stream, kwargs)
}
// Spawn goroutine for the async pipeline. All remaining phases run inside.
@@ -988,17 +988,15 @@ func (s *ChatPipelineService) AsyncChat(
return
}
- // Adjust max_tokens so the LLM has room within the total budget.
- if chat.LLMSetting != nil {
- if mt, ok := chat.LLMSetting["max_tokens"].(float64); ok {
- original := int(mt)
- adjusted := original
- if adjusted > modelMaxTokens-usedTokenCount {
- adjusted = modelMaxTokens - usedTokenCount
- }
- chat.LLMSetting["max_tokens"] = float64(adjusted)
- common.Debug("Adjusted max_tokens", zap.Int("max_tokens in chat", adjusted))
+ chatCfg := BuildChatConfig(chat, kwargs)
+ if adjusted, ok, err := clampChatConfigMaxTokens(chatCfg, modelMaxTokens, usedTokenCount); err != nil {
+ out <- AsyncChatResult{
+ Answer: fmt.Sprintf("**ERROR**: %s", err.Error()),
+ Final: true,
}
+ return
+ } else if ok {
+ common.Debug("Adjusted max_tokens", zap.Int("max_tokens in chat", adjusted))
}
// === Phase 11: Drive LLM + Decorate Answer ===
@@ -1061,8 +1059,6 @@ func (s *ChatPipelineService) AsyncChat(
var fullAnswer string
thinkState := &ThinkStreamState{}
- chatCfg := BuildChatConfig(chat, nil)
-
// Tool routing: use tool-loop method when tools are bound.
var driverErr error
if chatDriver.ToolConfig != nil {
@@ -1257,8 +1253,6 @@ func (s *ChatPipelineService) AsyncChat(
// Non-streaming: get the answer synchronously.
var answer string
var err error
- chatCfg := BuildChatConfig(chat, nil)
-
// Tool routing: use tool-loop when tools are bound.
if chatDriver.ToolConfig != nil {
answer, _, err = chatDriver.ChatWithTools(ctx, prompt+prompt4citation, chatMessages, chatCfg)
@@ -1310,6 +1304,7 @@ func (s *ChatPipelineService) AsyncChatSolo(
chat *entity.Chat,
messages []map[string]interface{},
stream bool,
+ config map[string]interface{},
) (<-chan AsyncChatResult, error) {
out := make(chan AsyncChatResult, 16)
@@ -1441,7 +1436,7 @@ func (s *ChatPipelineService) AsyncChatSolo(
if stream {
var fullAnswer string
thinkState := &ThinkStreamState{}
- chatCfg := BuildChatConfig(chat, nil)
+ chatCfg := BuildChatConfig(chat, config)
timer.Enter(common.PhaseGenerateAnswer)
driverErr := chatModel.ModelDriver.ChatStreamlyWithSender(
@@ -1568,7 +1563,7 @@ func (s *ChatPipelineService) AsyncChatSolo(
}
} else {
// Non-streaming: one-shot call.
- chatCfg := BuildChatConfig(chat, nil)
+ chatCfg := BuildChatConfig(chat, config)
timer.Enter(common.PhaseGenerateAnswer)
resp, err := chatModel.ModelDriver.ChatWithMessages(
ctx, *chatModel.ModelName, chatMessages, chatModel.APIConfig, chatCfg, nil,
@@ -4203,14 +4198,14 @@ func BuildChatConfig(dialog *entity.Chat, config map[string]interface{}) *modelM
if v, ok := dialog.LLMSetting["thinking"].(bool); ok {
cfg.Thinking = &v
}
- if v, ok := dialog.LLMSetting["max_tokens"].(float64); ok {
- i := int(v)
+ if v, ok := chatConfigPositiveInt(dialog.LLMSetting["max_tokens"]); ok {
+ i := v
cfg.MaxTokens = &i
}
- if v, ok := dialog.LLMSetting["temperature"].(float64); ok {
+ if v, ok := chatConfigFloat(dialog.LLMSetting["temperature"]); ok {
cfg.Temperature = &v
}
- if v, ok := dialog.LLMSetting["top_p"].(float64); ok {
+ if v, ok := chatConfigFloat(dialog.LLMSetting["top_p"]); ok {
cfg.TopP = &v
}
if v, ok := dialog.LLMSetting["do_sample"].(bool); ok {
@@ -4243,14 +4238,14 @@ func BuildChatConfig(dialog *entity.Chat, config map[string]interface{}) *modelM
if v, ok := config["thinking"].(bool); ok {
cfg.Thinking = &v
}
- if v, ok := config["max_tokens"].(float64); ok {
- i := int(v)
+ if v, ok := chatConfigPositiveInt(config["max_tokens"]); ok {
+ i := v
cfg.MaxTokens = &i
}
- if v, ok := config["temperature"].(float64); ok {
+ if v, ok := chatConfigFloat(config["temperature"]); ok {
cfg.Temperature = &v
}
- if v, ok := config["top_p"].(float64); ok {
+ if v, ok := chatConfigFloat(config["top_p"]); ok {
cfg.TopP = &v
}
if v, ok := config["do_sample"].(bool); ok {
@@ -4279,6 +4274,64 @@ func BuildChatConfig(dialog *entity.Chat, config map[string]interface{}) *modelM
return cfg
}
+func clampChatConfigMaxTokens(cfg *modelModule.ChatConfig, modelMaxTokens, usedTokenCount int) (int, bool, error) {
+ if usedTokenCount >= modelMaxTokens {
+ return 0, false, fmt.Errorf("prompt uses %d tokens, leaving no completion capacity for model max_tokens %d", usedTokenCount, modelMaxTokens)
+ }
+ if cfg == nil || cfg.MaxTokens == nil {
+ return 0, false, nil
+ }
+ adjusted := *cfg.MaxTokens
+ if adjusted <= 0 {
+ cfg.MaxTokens = nil
+ return 0, false, nil
+ }
+ remainingTokens := modelMaxTokens - usedTokenCount
+ if adjusted > remainingTokens {
+ adjusted = remainingTokens
+ }
+ cfg.MaxTokens = &adjusted
+ return adjusted, true, nil
+}
+
+func chatConfigPositiveInt(value interface{}) (int, bool) {
+ v, ok := chatConfigInt(value)
+ if !ok || v <= 0 {
+ return 0, false
+ }
+ return v, true
+}
+
+func chatConfigInt(value interface{}) (int, bool) {
+ switch v := value.(type) {
+ case int:
+ return v, true
+ case int64:
+ return int(v), true
+ case float64:
+ return int(v), true
+ case float32:
+ return int(v), true
+ default:
+ return 0, false
+ }
+}
+
+func chatConfigFloat(value interface{}) (float64, bool) {
+ switch v := value.(type) {
+ case float64:
+ return v, true
+ case float32:
+ return float64(v), true
+ case int:
+ return float64(v), true
+ case int64:
+ return float64(v), true
+ default:
+ return 0, false
+ }
+}
+
func kbIDStrings(kbs []*entity.Knowledgebase) []string {
if len(kbs) == 0 {
return nil
diff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.go
index e7c940babc..c02f197c10 100644
--- a/internal/service/chat_pipeline_test.go
+++ b/internal/service/chat_pipeline_test.go
@@ -26,6 +26,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/entity"
+ modelModule "ragflow/internal/entity/models"
)
// dialForTest builds a minimal *entity.Chat suitable for the
@@ -1442,9 +1443,13 @@ func TestBuildChatConfig_RequestOverrides(t *testing.T) {
LLMSetting: entity.JSONMap{
"temperature": 0.5,
"top_p": 0.9,
+ "max_tokens": float64(512),
},
}
- req := map[string]interface{}{"temperature": temp}
+ req := map[string]interface{}{
+ "temperature": temp,
+ "max_tokens": 128,
+ }
cfg := BuildChatConfig(dialog, req)
if cfg.Temperature == nil || *cfg.Temperature != temp {
t.Fatalf("expected request temperature %v, got %v", temp, cfg.Temperature)
@@ -1452,6 +1457,9 @@ func TestBuildChatConfig_RequestOverrides(t *testing.T) {
if cfg.TopP == nil || *cfg.TopP != 0.9 {
t.Fatalf("expected dialog top_p 0.9 to be preserved, got %v", cfg.TopP)
}
+ if cfg.MaxTokens == nil || *cfg.MaxTokens != 128 {
+ t.Fatalf("expected request max_tokens 128, got %v", cfg.MaxTokens)
+ }
}
// TestBuildChatConfig_FromEmptyDialog verifies the merger works even when
@@ -1465,3 +1473,70 @@ func TestBuildChatConfig_FromEmptyDialog(t *testing.T) {
t.Fatalf("expected temperature %v, got %v", temp, cfg.Temperature)
}
}
+
+func TestClampChatConfigMaxTokensUsesRemainingBudget(t *testing.T) {
+ dialog := &entity.Chat{
+ LLMSetting: entity.JSONMap{"max_tokens": float64(1024)},
+ }
+ cfg := BuildChatConfig(dialog, map[string]interface{}{"max_tokens": 700})
+
+ adjusted, ok, err := clampChatConfigMaxTokens(cfg, 1000, 450)
+ if err != nil {
+ t.Fatalf("clampChatConfigMaxTokens returned error: %v", err)
+ }
+ if !ok {
+ t.Fatal("expected max_tokens to be clamped")
+ }
+ if adjusted != 550 {
+ t.Fatalf("adjusted max_tokens = %d, want 550", adjusted)
+ }
+ if cfg.MaxTokens == nil || *cfg.MaxTokens != 550 {
+ t.Fatalf("config max_tokens = %v, want 550", cfg.MaxTokens)
+ }
+ if dialog.LLMSetting["max_tokens"] != float64(1024) {
+ t.Fatalf("dialog max_tokens was mutated: %v", dialog.LLMSetting["max_tokens"])
+ }
+}
+
+func TestBuildChatConfigIgnoresNonPositiveMaxTokens(t *testing.T) {
+ dialog := &entity.Chat{
+ LLMSetting: entity.JSONMap{"max_tokens": float64(0)},
+ }
+ cfg := BuildChatConfig(dialog, nil)
+ if cfg.MaxTokens != nil {
+ t.Fatalf("persisted max_tokens=0 must not populate ChatConfig, got %v", *cfg.MaxTokens)
+ }
+
+ cfg = BuildChatConfig(&entity.Chat{}, map[string]interface{}{"max_tokens": -1})
+ if cfg.MaxTokens != nil {
+ t.Fatalf("request max_tokens=-1 must not populate ChatConfig, got %v", *cfg.MaxTokens)
+ }
+}
+
+func TestClampChatConfigMaxTokensRejectsNonPositive(t *testing.T) {
+ for _, value := range []int{0, -1} {
+ cfg := &modelModule.ChatConfig{MaxTokens: &value}
+ if adjusted, ok, err := clampChatConfigMaxTokens(cfg, 1000, 100); err != nil {
+ t.Fatalf("max_tokens=%d returned unexpected error: %v", value, err)
+ } else if ok {
+ t.Fatalf("max_tokens=%d unexpectedly clamped to %d", value, adjusted)
+ }
+ if cfg.MaxTokens != nil {
+ t.Fatalf("max_tokens=%d should be cleared before provider request, got %v", value, *cfg.MaxTokens)
+ }
+ }
+}
+
+func TestClampChatConfigMaxTokensRejectsExhaustedCapacity(t *testing.T) {
+ for _, usedTokenCount := range []int{1000, 1001} {
+ maxTokens := 700
+ cfg := &modelModule.ChatConfig{MaxTokens: &maxTokens}
+
+ if adjusted, ok, err := clampChatConfigMaxTokens(cfg, 1000, usedTokenCount); err == nil {
+ t.Fatalf("usedTokenCount=%d expected capacity error, got adjusted=%d ok=%t", usedTokenCount, adjusted, ok)
+ }
+ if cfg.MaxTokens == nil || *cfg.MaxTokens != maxTokens {
+ t.Fatalf("capacity error should not set max_tokens to zero, got %v", cfg.MaxTokens)
+ }
+ }
+}
diff --git a/internal/service/openai_chat.go b/internal/service/openai_chat.go
index 1c0125e11f..31dfc2b079 100644
--- a/internal/service/openai_chat.go
+++ b/internal/service/openai_chat.go
@@ -153,6 +153,10 @@ func (s *OpenAIChatService) OpenAIChatCompletions(c *gin.Context, userID, chatID
s.writeDataError(c, "You have to provide messages.")
return
}
+ if req.MaxTokens != nil && *req.MaxTokens <= 0 {
+ s.writeArgError(c, "`max_tokens` must be greater than 0.")
+ return
+ }
lastRole, _ := normalizedMessages[len(normalizedMessages)-1]["role"].(string)
if lastRole != "user" {
diff --git a/internal/service/openai_chat_test.go b/internal/service/openai_chat_test.go
index 96a115bef4..f3e18e889e 100644
--- a/internal/service/openai_chat_test.go
+++ b/internal/service/openai_chat_test.go
@@ -136,12 +136,8 @@ func TestOpenAI_MergeGenerationConfig_AddsNewKeys(t *testing.T) {
// calls BuildChatConfig(dialog, nil) which reads the merged values.
// Verifies that all 5 fields the Python server honors
// (temperature, top_p, max_tokens, frequency_penalty, presence_penalty)
-// survive the merge. For 3 of them (temperature, top_p, max_tokens) the
-// ChatConfig fields exist and the test asserts the values. For the
-// other 2 (frequency_penalty, presence_penalty) the ChatConfig struct
-// doesn't have fields yet, so we just assert the dialog's LLMSetting
-// preserves them — the structural gap is documented in
-// openai_chat_completions.go::extractGenerationConfig.
+// survive the merge. For the fields ChatConfig supports, the values must
+// surface on the returned struct.
//
// The handler-side float64 coercion of max_tokens is verified by
// TestExtractGenerationConfig_OnlyKnownFields in the handler package.
@@ -178,9 +174,6 @@ func TestOpenAI_MergeThenBuild_AllGenerationParamsReachChatConfig(t *testing.T)
t.Fatalf("presence_penalty: expected 0.2, got %v", got)
}
- // Now run the RAG-pipeline call: BuildChatConfig(dialog, nil).
- // For the 3 fields ChatConfig supports, the values must surface
- // on the returned struct.
cfg := BuildChatConfig(dialog, nil)
if cfg.Temperature == nil || *cfg.Temperature != 0.7 {
t.Fatalf("ChatConfig.Temperature: expected 0.7, got %v", cfg.Temperature)
diff --git a/web/src/interfaces/database/chat.ts b/web/src/interfaces/database/chat.ts
index 5a76003ec5..d3170fc073 100644
--- a/web/src/interfaces/database/chat.ts
+++ b/web/src/interfaces/database/chat.ts
@@ -48,6 +48,7 @@ export interface Variable {
presence_penalty?: number;
temperature?: number;
top_p?: number;
+ thinking?: 'default' | 'enabled' | 'disabled';
tenant_llm_id?: string;
model_type?: string;
}
diff --git a/web/src/pages/next-chats/chat-stream/run-stream.ts b/web/src/pages/next-chats/chat-stream/run-stream.ts
index fbf9ec644d..25c2227c8d 100644
--- a/web/src/pages/next-chats/chat-stream/run-stream.ts
+++ b/web/src/pages/next-chats/chat-stream/run-stream.ts
@@ -3,7 +3,7 @@
* keep writing into the store) after the chat page unmounts, so that navigating
* away mid-answer and back shows the stream still in progress.
*/
-import { IMessage } from '@/interfaces/database/chat';
+import { IMessage, Variable } from '@/interfaces/database/chat';
import {
CompletionChunk,
parseCompletionEventStream,
@@ -20,6 +20,7 @@ export type RunChatCompletionStreamParams = {
messages: IMessage[];
enableThinking?: string;
enableInternet?: boolean;
+ llmSetting?: Variable;
};
export type RunChatCompletionStreamResult = {
@@ -33,6 +34,7 @@ export async function runChatCompletionStream({
messages,
enableThinking,
enableInternet,
+ llmSetting,
}: RunChatCompletionStreamParams): Promise<RunChatCompletionStreamResult> {
const { beginStream, applyAnswer, endStream } = useChatStreamStore.getState();
@@ -76,6 +78,7 @@ export async function runChatCompletionStream({
messages,
enableThinking,
enableInternet,
+ llmSetting,
},
controller.signal,
);
diff --git a/web/src/pages/next-chats/hooks/use-send-chat-message.ts b/web/src/pages/next-chats/hooks/use-send-chat-message.ts
index d94f4da4ed..5104407728 100644
--- a/web/src/pages/next-chats/hooks/use-send-chat-message.ts
+++ b/web/src/pages/next-chats/hooks/use-send-chat-message.ts
@@ -4,7 +4,7 @@ import {
useHandleMessageInputChange,
useScrollToBottom,
} from '@/hooks/logic-hooks';
-import { useGetChatSearchParams } from '@/hooks/use-chat-request';
+import { useFetchChat, useGetChatSearchParams } from '@/hooks/use-chat-request';
import { buildMessageListWithUuid } from '@/utils/chat';
import { IMessage, Message } from '@/interfaces/database/chat';
import notification from '@/utils/notification';
@@ -72,6 +72,7 @@ export const useSendMessage = () => {
useUploadFile();
const { id: chatId } = useParams();
+ const { data: currentDialog } = useFetchChat();
const { messages, isStreaming, scrollRef, messageContainerRef } =
useCurrentChatSession();
@@ -121,6 +122,7 @@ export const useSendMessage = () => {
],
enableThinking,
enableInternet,
+ llmSetting: currentDialog?.llm_setting,
});
if (!ok && !aborted) {
@@ -132,7 +134,14 @@ export const useSendMessage = () => {
notification.error({ message: t('message.requestError') });
}
},
- [conversationId, chatId, messages, failStream, t],
+ [
+ conversationId,
+ chatId,
+ messages,
+ failStream,
+ t,
+ currentDialog?.llm_setting,
+ ],
);
// Hand a failed question back to the input box, but only once the box is
diff --git a/web/src/services/chat-completion-stream.ts b/web/src/services/chat-completion-stream.ts
index da9e313c9c..58b242daee 100644
--- a/web/src/services/chat-completion-stream.ts
+++ b/web/src/services/chat-completion-stream.ts
@@ -11,7 +11,7 @@
*/
import { Authorization } from '@/constants/authorization';
import { ResponseType } from '@/interfaces/database/base';
-import { IMessage } from '@/interfaces/database/chat';
+import { IMessage, Variable } from '@/interfaces/database/chat';
import api from '@/utils/api';
import { getAuthorization } from '@/utils/authorization-util';
import { EventSourceParserStream } from 'eventsource-parser/stream';
@@ -22,6 +22,7 @@ export type ChatCompletionStreamParams = {
messages: IMessage[];
enableThinking?: string;
enableInternet?: boolean;
+ llmSetting?: Variable;
};
export type CompletionChunk = {
@@ -32,10 +33,32 @@ export type CompletionChunk = {
[key: string]: any;
};
+function normalizeThinkingForRequest(thinking?: Variable['thinking']) {
+ if (thinking === 'enabled' || thinking === 'disabled') return thinking;
+ return undefined;
+}
+
export function requestChatCompletionStream(
- { chatId, sessionId, messages, enableThinking, enableInternet }: ChatCompletionStreamParams,
+ {
+ chatId,
+ sessionId,
+ messages,
+ enableThinking,
+ enableInternet,
+ llmSetting,
+ }: ChatCompletionStreamParams,
signal: AbortSignal,
) {
+ const {
+ temperature,
+ top_p,
+ frequency_penalty,
+ presence_penalty,
+ max_tokens,
+ thinking,
+ } = llmSetting ?? {};
+ const requestThinking = normalizeThinkingForRequest(thinking);
+
return fetch(api.completionUrl, {
method: 'POST',
headers: {
@@ -49,6 +72,12 @@ export function requestChatCompletionStream(
pass_all_history_messages: true,
reasoning: Number(enableThinking),
internet: enableInternet,
+ ...(temperature === undefined ? {} : { temperature }),
+ ...(top_p === undefined ? {} : { top_p }),
+ ...(frequency_penalty === undefined ? {} : { frequency_penalty }),
+ ...(presence_penalty === undefined ? {} : { presence_penalty }),
+ ...(max_tokens === undefined ? {} : { max_tokens }),
+ ...(requestThinking === undefined ? {} : { thinking: requestThinking }),
}),
signal,
});