Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: add configurable LLM streaming output

nash_su4fcaef6992de
diff --git a/src-tauri/src/agent/provider.rs b/src-tauri/src/agent/provider.rs
index fb895c0b..b40f9371 100644
--- a/src-tauri/src/agent/provider.rs
+++ b/src-tauri/src/agent/provider.rs
@@ -53,6 +53,9 @@ pub struct LlmConfig {
/// each request because app-state may be edited outside the settings UI.
#[serde(default)]
pub custom_headers: BTreeMap<String, String>,
+ /// Missing keeps the historical streaming behavior for existing configs.
+ #[serde(default)]
+ pub streaming_enabled: Option<bool>,
}
impl LlmConfig {
@@ -211,11 +214,16 @@ impl LlmClient {
system: &str,
user: &str,
images: &[AgentImage],
- on_delta: F,
+ mut on_delta: F,
) -> Result<String, String>
where
F: FnMut(&str) + Send,
{
+ if self.config.streaming_enabled == Some(false) {
+ let text = self.generate_text(system, user, images).await?;
+ on_delta(&text);
+ return Ok(text);
+ }
match self.config.provider.as_str() {
"openai" => {
self.stream_openai_like(
@@ -991,6 +999,7 @@ mod tests {
max_tokens: None,
max_context_size: None,
custom_headers: BTreeMap::new(),
+ streaming_enabled: None,
}
}
@@ -1092,6 +1101,14 @@ mod tests {
Some(4096)
);
assert_eq!(cfg.max_tokens, None);
+ assert_eq!(cfg.streaming_enabled, None);
+
+ let disabled: LlmConfig = serde_json::from_value(json!({
+ "provider": "openai",
+ "streamingEnabled": false
+ }))
+ .unwrap();
+ assert_eq!(disabled.streaming_enabled, Some(false));
}
#[test]
diff --git a/src-tauri/src/agent/runtime.rs b/src-tauri/src/agent/runtime.rs
index 8afff847..0bcbc51f 100644
--- a/src-tauri/src/agent/runtime.rs
+++ b/src-tauri/src/agent/runtime.rs
@@ -4176,6 +4176,7 @@ mod tests {
max_tokens: None,
max_context_size: Some(8_000),
custom_headers: Default::default(),
+ streaming_enabled: None,
};
let fitted = fit_context_to_model(context, Some(&config));
assert!(fitted.system.contains("system"));
@@ -4201,6 +4202,7 @@ mod tests {
max_tokens: None,
max_context_size: Some(8_000),
custom_headers: Default::default(),
+ streaming_enabled: None,
};
let fitted = fit_context_to_model(context, Some(&config));
diff --git a/src-tauri/src/api_server.rs b/src-tauri/src/api_server.rs
index 37665d38..0d8ebb83 100644
--- a/src-tauri/src/api_server.rs
+++ b/src-tauri/src/api_server.rs
@@ -1851,6 +1851,7 @@ fn project_llm_config(parsed: &Value, project_id: &str) -> Option<agent::provide
"azureApiVersion",
"maxContextSize",
"reasoning",
+ "streamingEnabled",
"customHeaders",
] {
if let Some(value) = provider.get(key) {
@@ -2808,7 +2809,7 @@ mod tests {
let state = json!({
"llmConfig": { "provider": "openai", "apiKey": "global", "model": "gpt-global", "ollamaUrl": "", "customEndpoint": "", "maxContextSize": 1000 },
"providerConfigs": {
- "deepseek": { "apiKey": "rotated", "model": "provider-model", "baseUrl": "https://new.example/v1", "apiMode": "chat_completions", "customHeaders": { "X-Tenant-ID": "team-a" } }
+ "deepseek": { "apiKey": "rotated", "model": "provider-model", "baseUrl": "https://new.example/v1", "apiMode": "chat_completions", "streamingEnabled": false, "customHeaders": { "X-Tenant-ID": "team-a" } }
},
"projectLlmOverrides": {
"project-a": {
@@ -2824,6 +2825,7 @@ mod tests {
assert_eq!(config.api_key, "rotated");
assert_eq!(config.model, "project-model");
assert_eq!(config.custom_endpoint, "https://new.example/v1");
+ assert_eq!(config.streaming_enabled, Some(false));
assert_eq!(
config.custom_headers.get("X-Tenant-ID").map(String::as_str),
Some("team-a")
diff --git a/src/components/settings/preset-resolver.test.ts b/src/components/settings/preset-resolver.test.ts
index f8e17cb7..73520da1 100644
--- a/src/components/settings/preset-resolver.test.ts
+++ b/src/components/settings/preset-resolver.test.ts
@@ -94,6 +94,27 @@ describe("resolveConfig", () => {
expect(resolved.reasoning).toEqual({ mode: "off" })
})
+ it("preserves an explicit non-streaming provider preference", () => {
+ const preset: LlmPreset = {
+ id: "openai",
+ label: "OpenAI",
+ provider: "openai",
+ defaultModel: "gpt-5",
+ }
+
+ expect(resolveConfig(
+ preset,
+ { streamingEnabled: false },
+ fallbackConfig(),
+ ).streamingEnabled).toBe(false)
+ expect(resolveConfig(
+ preset,
+ undefined,
+ fallbackConfig({ streamingEnabled: false }),
+ ).streamingEnabled)
+ .toBeUndefined()
+ })
+
it("carries Azure API version and model family overrides", () => {
const preset: LlmPreset = {
id: "azure",
diff --git a/src/components/settings/preset-resolver.ts b/src/components/settings/preset-resolver.ts
index a599d049..47d09cce 100644
--- a/src/components/settings/preset-resolver.ts
+++ b/src/components/settings/preset-resolver.ts
@@ -45,6 +45,11 @@ export function resolveConfig(
? Math.max(1, Math.min(1440, Math.floor(ov.requestTimeoutMinutes)))
: fallback.requestTimeoutMinutes
const customHeaders = ov.customHeaders
+ // Streaming is a per-preset preference. Never inherit it from the currently
+ // active fallback preset, or a newly selected provider would silently adopt
+ // the previous provider's disabled state. Missing means legacy/default on.
+ const streamingEnabled = ov.streamingEnabled
+ const streamingConfig = streamingEnabled === undefined ? {} : { streamingEnabled }
if (preset.provider === "custom") {
return {
@@ -59,6 +64,7 @@ export function resolveConfig(
localCliIsolation: false,
requestTimeoutMinutes,
customHeaders,
+ ...streamingConfig,
}
}
@@ -74,6 +80,7 @@ export function resolveConfig(
localCliIsolation: false,
requestTimeoutMinutes,
customHeaders,
+ ...streamingConfig,
}
}
@@ -91,6 +98,7 @@ export function resolveConfig(
localCliIsolation: false,
requestTimeoutMinutes,
customHeaders,
+ ...streamingConfig,
}
}
@@ -108,6 +116,7 @@ export function resolveConfig(
localCliIsolation,
codexCliTimeoutMinutes: preset.provider === "codex-cli" ? codexCliTimeoutMinutes : undefined,
requestTimeoutMinutes,
+ ...streamingConfig,
}
}
@@ -125,5 +134,6 @@ export function resolveConfig(
localCliIsolation: false,
requestTimeoutMinutes,
customHeaders,
+ ...streamingConfig,
}
}
diff --git a/src/components/settings/sections/llm-provider-section.tsx b/src/components/settings/sections/llm-provider-section.tsx
index c2e4198b..092ff5bb 100644
--- a/src/components/settings/sections/llm-provider-section.tsx
+++ b/src/components/settings/sections/llm-provider-section.tsx
@@ -382,11 +382,12 @@ function PresetRow({
const localCliIsolation = ov.localCliIsolation === true
const codexCliTimeoutMinutes = Math.max(1, Math.min(240, ov.codexCliTimeoutMinutes ?? 10))
const requestTimeoutMinutes = Math.max(1, Math.min(1440, ov.requestTimeoutMinutes ?? 30))
+ const streamingEnabled = ov.streamingEnabled !== false
const [headersText, setHeadersText] = useState(() => llmHeadersToText(ov.customHeaders))
const isLocalCliProvider = preset.provider === "claude-code" || preset.provider === "codex-cli"
const [testState, setTestState] = useState<ProviderTestState>({ kind: "idle" })
const hasConfig = !!apiKey || !!ov.baseUrl || !!ov.model || !!ov.azureApiVersion || !!ov.azureModelFamily
- || Object.keys(ov.customHeaders ?? {}).length > 0
+ || Object.keys(ov.customHeaders ?? {}).length > 0 || ov.streamingEnabled === false
// Local CLI providers authenticate via their own existing login state
// (inherited by the spawned subprocess), so no API key field is shown.
// Ollama ditto for its local-only model.
@@ -688,6 +689,45 @@ function PresetRow({
/>
</div>
+ <div className="space-y-2 rounded-md border p-3">
+ <div className="flex items-start justify-between gap-3">
+ <div>
+ <div className="text-sm font-medium">
+ {t("settings.sections.llm.streamingOutput")}
+ </div>
+ <p className="mt-1 text-xs text-muted-foreground">
+ {t("settings.sections.llm.streamingOutputHint")}
+ </p>
+ </div>
+ <button
+ type="button"
+ role="switch"
+ aria-checked={streamingEnabled}
+ onClick={() => onChange({ streamingEnabled: !streamingEnabled })}
+ className={`relative inline-flex h-5 w-9 shrink-0 items-center rounded-full border transition-colors ${
+ streamingEnabled
+ ? "border-primary bg-primary"
+ : "border-muted-foreground/30 bg-muted-foreground/20 hover:bg-muted-foreground/30"
+ }`}
+ title={streamingEnabled
+ ? t("settings.sections.llm.streamingOutputOn")
+ : t("settings.sections.llm.streamingOutputOff")}
+ aria-label={t("settings.sections.llm.streamingOutput")}
+ >
+ <span
+ className={`inline-block h-3.5 w-3.5 rounded-full bg-white shadow-sm ring-1 ring-black/10 transition-transform ${
+ streamingEnabled ? "translate-x-4" : "translate-x-0.5"
+ }`}
+ />
+ </button>
+ </div>
+ <div className="rounded-md bg-muted/50 px-2 py-1.5 text-xs text-muted-foreground">
+ {streamingEnabled
+ ? t("settings.sections.llm.streamingOutputOn")
+ : t("settings.sections.llm.streamingOutputOff")}
+ </div>
+ </div>
+
{!isLocalCliProvider && (
<>
<div className="space-y-2">
diff --git a/src/i18n/en.json b/src/i18n/en.json
index c7c16c31..24506088 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -521,6 +521,10 @@
"codexCliTimeoutHint": "Overall timeout for one Codex CLI request. Raise this for long PDF ingest or slower local/network sessions.",
"requestTimeout": "Request timeout (minutes)",
"requestTimeoutHint": "Increase this for slow local CPU models. The default is 30 minutes.",
+ "streamingOutput": "Stream responses",
+ "streamingOutputHint": "When disabled, HTTP providers wait for the complete response before displaying it. CLI providers still use their native protocol but buffer visible output until completion.",
+ "streamingOutputOn": "Streaming on: response text appears as it is generated.",
+ "streamingOutputOff": "Streaming off: response text appears after generation completes.",
"customHeaders": "Custom request headers",
"customHeadersHint": "Optional. Enter one Header-Name: value per line. Content-Type and configured API-key headers take precedence.",
"cliStatus": "CLI status",
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index 3c789fc4..fd202ae9 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -521,6 +521,10 @@
"codexCliTimeoutHint": "单次 Codex CLI 请求的总超时时间。长 PDF 摄取或较慢的本地 / 网络会话可适当调大。",
"requestTimeout": "请求超时时间(分钟)",
"requestTimeoutHint": "本地 CPU 模型响应较慢时可适当调高,默认 30 分钟。",
+ "streamingOutput": "启用流式输出",
+ "streamingOutputHint": "关闭后,HTTP Provider 会等待完整响应后再显示。CLI Provider 仍使用其原生协议,但会缓存可见内容并在完成后一次性显示。",
+ "streamingOutputOn": "流式输出已开启:生成内容会逐步显示。",
+ "streamingOutputOff": "流式输出已关闭:生成完成后一次性显示内容。",
"customHeaders": "自定义请求头",
"customHeadersHint": "可选。每行填写一条 Header-Name: value。Content-Type 和已配置的 API 密钥请求头优先。",
"cliStatus": "CLI 状态",
diff --git a/src/lib/llm-client.test.ts b/src/lib/llm-client.test.ts
index 56c18f3a..5b4e9261 100644
--- a/src/lib/llm-client.test.ts
+++ b/src/lib/llm-client.test.ts
@@ -91,6 +91,49 @@ const cfg: LlmConfig = {
maxContextSize: 8192,
}
+describe("streamChat — non-streaming HTTP responses", () => {
+ beforeEach(() => mockHttpFetch.mockReset())
+
+ it("emits one complete token and completes", async () => {
+ mockHttpFetch.mockResolvedValue(new Response(JSON.stringify({
+ choices: [{ message: { content: "complete answer" } }],
+ }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ }))
+ const onToken = vi.fn()
+ const onDone = vi.fn()
+ const onError = vi.fn()
+
+ await streamChat(
+ { ...cfg, streamingEnabled: false },
+ [{ role: "user", content: "hi" }],
+ { onToken, onDone, onError },
+ )
+
+ expect(onToken).toHaveBeenCalledTimes(1)
+ expect(onToken).toHaveBeenCalledWith("complete answer")
+ expect(onDone).toHaveBeenCalledTimes(1)
+ expect(onError).not.toHaveBeenCalled()
+ expect(JSON.parse(String(mockHttpFetch.mock.calls[0][1]?.body))).toMatchObject({ stream: false })
+ })
+
+ it("reports an empty complete response instead of silently succeeding", async () => {
+ mockHttpFetch.mockResolvedValue(new Response(JSON.stringify({
+ choices: [{ message: { content: "" } }],
+ }), { status: 200 }))
+ const onError = vi.fn()
+
+ await streamChat(
+ { ...cfg, streamingEnabled: false },
+ [{ role: "user", content: "hi" }],
+ { onToken: vi.fn(), onDone: vi.fn(), onError },
+ )
+
+ expect(onError.mock.calls[0][0].message).toContain("empty non-streaming response")
+ })
+})
+
/** A Response whose reader.read() stays pending until we reject it,
* letting the test interleave the 30-min backstop before the abort.
* `readCalled` resolves once streamChat reaches read(), so the test
diff --git a/src/lib/llm-client.ts b/src/lib/llm-client.ts
index 34bcc3b0..a4d9ab5c 100644
--- a/src/lib/llm-client.ts
+++ b/src/lib/llm-client.ts
@@ -14,6 +14,21 @@ export interface StreamCallbacks {
onError: (error: Error) => void
}
+function bufferedStreamCallbacks(callbacks: StreamCallbacks): StreamCallbacks {
+ let content = ""
+ let reasoning = ""
+ return {
+ onToken: (token) => { content += token },
+ onReasoningToken: (token) => { reasoning += token },
+ onDone: () => {
+ if (reasoning) callbacks.onReasoningToken?.(reasoning)
+ if (content) callbacks.onToken(content)
+ callbacks.onDone()
+ },
+ onError: callbacks.onError,
+ }
+}
+
// Lazy import keeps the Tauri event/invoke bindings out of bundles that
// never touch the subprocess provider (e.g. vitest with a fetch mock).
async function streamViaClaudeCodeCli(
@@ -78,11 +93,23 @@ export async function streamChat(
// HTTP. Dispatch before getProviderConfig — that function throws for
// this provider because it has no URL/headers.
if (config.provider === "claude-code") {
- return streamViaClaudeCodeCli(config, messages, callbacks, signal, requestOverrides)
+ return streamViaClaudeCodeCli(
+ config,
+ messages,
+ config.streamingEnabled === false ? bufferedStreamCallbacks(callbacks) : callbacks,
+ signal,
+ requestOverrides,
+ )
}
if (config.provider === "codex-cli") {
- return streamViaCodexCli(config, messages, callbacks, signal, requestOverrides)
+ return streamViaCodexCli(
+ config,
+ messages,
+ config.streamingEnabled === false ? bufferedStreamCallbacks(callbacks) : callbacks,
+ signal,
+ requestOverrides,
+ )
}
const providerConfig = getProviderConfig(config)
@@ -183,6 +210,38 @@ export async function streamChat(
return
}
+ if (!providerConfig.streaming) {
+ try {
+ const payload: unknown = await response.json()
+ const content = providerConfig.parseResponse(payload)
+ if (!content) {
+ onError(new Error("Model returned an empty non-streaming response"))
+ return
+ }
+ onToken(content)
+ onDone()
+ } catch (err) {
+ if (timeoutFired) {
+ onError(new Error(`Request timed out after ${Math.round(timeoutMs / 60000)} min. Try a faster model or a smaller context.`))
+ return
+ }
+ if (
+ signal?.aborted ||
+ (err instanceof Error && err.name === "AbortError") ||
+ isRequestCancelledError(err)
+ ) {
+ onDone()
+ return
+ }
+ if (isFetchNetworkError(err)) {
+ onError(new Error("Connection lost while reading the complete response. Try again."))
+ return
+ }
+ onError(err instanceof Error ? err : new Error(String(err)))
+ }
+ return
+ }
+
if (!response.body) {
onError(new Error("Response body is null"))
return
diff --git a/src/lib/llm-providers.test.ts b/src/lib/llm-providers.test.ts
index 885fd971..299767f2 100644
--- a/src/lib/llm-providers.test.ts
+++ b/src/lib/llm-providers.test.ts
@@ -19,6 +19,9 @@ import { describe, it, expect } from "vitest"
import {
buildAnthropicUrl,
getProviderConfig,
+ parseAnthropicResponse,
+ parseGoogleResponse,
+ parseOpenAiResponse,
supportsImageInput,
type ChatMessage,
type ContentBlock,
@@ -40,6 +43,52 @@ function mkConfig(over: Partial<LlmConfig>): LlmConfig {
}
}
+describe("non-streaming provider configuration", () => {
+ it("disables streaming on OpenAI and Anthropic request bodies", () => {
+ const messages: ChatMessage[] = [{ role: "user", content: "hello" }]
+ const openai = getProviderConfig(mkConfig({ streamingEnabled: false }))
+ const anthropic = getProviderConfig(mkConfig({
+ provider: "anthropic",
+ model: "claude-3-5-sonnet-latest",
+ streamingEnabled: false,
+ }))
+
+ expect(openai.streaming).toBe(false)
+ expect(openai.buildBody(messages)).toMatchObject({ stream: false })
+ expect(anthropic.streaming).toBe(false)
+ expect(anthropic.buildBody(messages)).toMatchObject({ stream: false })
+ })
+
+ it("uses Gemini generateContent instead of the SSE endpoint", () => {
+ const provider = getProviderConfig(mkConfig({
+ provider: "google",
+ model: "gemini-2.5-flash",
+ streamingEnabled: false,
+ }))
+
+ expect(provider.url).toContain(":generateContent")
+ expect(provider.url).not.toContain("streamGenerateContent")
+ })
+
+ it("extracts complete response text for each HTTP wire", () => {
+ expect(parseOpenAiResponse({ choices: [{ message: { content: "openai" } }] })).toBe("openai")
+ expect(parseAnthropicResponse({ content: [{ type: "text", text: "anthropic" }] })).toBe("anthropic")
+ expect(parseGoogleResponse({
+ candidates: [{ content: { parts: [
+ { text: "hidden", thought: true },
+ { text: "google" },
+ ] } }],
+ })).toBe("google")
+ })
+
+ it("keeps streaming enabled for legacy configs without the new field", () => {
+ const provider = getProviderConfig(mkConfig({}))
+ expect(provider.streaming).toBe(true)
+ expect(provider.buildBody([{ role: "user", content: "hello" }]))
+ .toMatchObject({ stream: true })
+ })
+})
+
function visionMessage(): ChatMessage {
const blocks: ContentBlock[] = [
{ type: "text", text: "What's in this image?" },
diff --git a/src/lib/llm-providers.ts b/src/lib/llm-providers.ts
index a8239dca..14f47e7d 100644
--- a/src/lib/llm-providers.ts
+++ b/src/lib/llm-providers.ts
@@ -63,6 +63,8 @@ interface ProviderConfig {
headers: Record<string, string>
buildBody: (messages: ChatMessage[], overrides?: RequestOverrides) => unknown
parseStream: (line: string) => string | null
+ parseResponse: (payload: unknown) => string
+ streaming: boolean
}
const JSON_CONTENT_TYPE = "application/json"
@@ -177,6 +179,38 @@ function parseOpenAiLine(line: string): string | null {
}
}
+function textFromUnknownContent(content: unknown): string {
+ if (typeof content === "string") return content
+ if (!Array.isArray(content)) return ""
+ return content
+ .map((part) => {
+ if (!part || typeof part !== "object") return ""
+ const value = part as Record<string, unknown>
+ return typeof value.text === "string" ? value.text : ""
+ })
+ .join("")
+}
+
+export function parseOpenAiResponse(payload: unknown): string {
+ const root = payload as { choices?: Array<{ message?: { content?: unknown } }> }
+ return textFromUnknownContent(root?.choices?.[0]?.message?.content)
+}
+
+export function parseAnthropicResponse(payload: unknown): string {
+ const root = payload as { content?: unknown }
+ return textFromUnknownContent(root?.content)
+}
+
+export function parseGoogleResponse(payload: unknown): string {
+ const root = payload as {
+ candidates?: Array<{ content?: { parts?: Array<{ text?: string; thought?: boolean }> } }>
+ }
+ return (root?.candidates?.[0]?.content?.parts ?? [])
+ .filter((part) => !part.thought)
+ .map((part) => part.text ?? "")
+ .join("")
+}
+
export function parseAnthropicLine(line: string): string | null {
if (!line.startsWith("data:")) return null
const data = line.slice(5).trim()
@@ -284,6 +318,7 @@ function toOpenAiContent(content: string | ContentBlock[]): unknown {
function buildOpenAiBody(
messages: ChatMessage[],
overrides?: RequestOverrides,
+ streaming = true,
): Record<string, unknown> {
// OpenAI (and every /v1/chat/completions clone — DeepSeek, Groq,
// Ollama, Zhipu, Kimi, xAI, MiniMax OpenAI-compat, ...) accepts these
@@ -292,7 +327,7 @@ function buildOpenAiBody(
role: m.role,
content: toOpenAiContent(m.content),
}))
- return { messages: translated, stream: true, ...stripWireAgnosticOverrides(overrides) }
+ return { messages: translated, stream: streaming, ...stripWireAgnosticOverrides(overrides) }
}
function stripWireAgnosticOverrides(overrides?: RequestOverrides): Omit<RequestOverrides, "reasoning"> {
@@ -413,10 +448,15 @@ function buildOpenAiCompatibleBody(
config: LlmConfig,
messages: ChatMessage[],
overrides?: RequestOverrides,
+ streaming = true,
): Record<string, unknown> {
assertBigModelImageSupport(config, messages)
const reasoning = effectiveReasoning(config, overrides)
- const body: Record<string, unknown> = buildOpenAiBody(messages, stripWireAgnosticOverrides(overrides))
+ const body: Record<string, unknown> = buildOpenAiBody(
+ messages,
+ stripWireAgnosticOverrides(overrides),
+ streaming,
+ )
adaptOpenAiStrictCompletionBody(config, body)
adaptKimiBody(config, body)
adaptXiaomiMimoBody(config, body, reasoning)
@@ -540,6 +580,7 @@ function buildAnthropicSystem(systemText: string): unknown[] | undefined {
function buildAnthropicBody(
messages: ChatMessage[],
overrides?: RequestOverrides,
+ streaming = true,
): Record<string, unknown> {
const systemMessages = messages.filter((m) => m.role === "system")
const conversationMessages = messages
@@ -556,7 +597,7 @@ function buildAnthropicBody(
return {
messages: conversationMessages,
...(system !== undefined ? { system } : {}),
- stream: true,
+ stream: streaming,
max_tokens: overrides?.max_tokens ?? 4096,
...(overrides?.temperature !== undefined ? { temperature: overrides.temperature } : {}),
...(overrides?.top_p !== undefined ? { top_p: overrides.top_p } : {}),
@@ -571,8 +612,9 @@ function buildAnthropicBodyWithReasoning(
config: LlmConfig,
messages: ChatMessage[],
overrides?: RequestOverrides,
+ streaming = true,
): Record<string, unknown> {
- const body = buildAnthropicBody(messages, overrides)
+ const body = buildAnthropicBody(messages, overrides, streaming)
const reasoning = effectiveReasoning(config, overrides)
if (reasoning.mode === "auto" || reasoning.mode === "off") return body
@@ -838,6 +880,7 @@ export function deriveAnthropicMaxTokens(maxContextSize: number | undefined): nu
export function getProviderConfig(config: LlmConfig): ProviderConfig {
const { provider, apiKey, model, ollamaUrl, customEndpoint } = config
+ const streaming = config.streamingEnabled !== false
// Default max_tokens for Anthropic-wire providers, derived from the
// configured context window rather than a hardcoded 4096. Converts the
@@ -856,10 +899,12 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
Authorization: `Bearer ${apiKey}`,
}),
buildBody: (messages, overrides) => ({
- ...buildOpenAiCompatibleBody(config, messages, overrides),
+ ...buildOpenAiCompatibleBody(config, messages, overrides, streaming),
model,
}),
parseStream: parseOpenAiLine,
+ parseResponse: parseOpenAiResponse,
+ streaming,
}
case "anthropic": {
@@ -870,11 +915,13 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
buildBody: (messages, overrides) => {
assertMiniMaxImageSupport(url, model, messages)
return {
- ...buildAnthropicBodyWithReasoning(config, messages, { max_tokens: anthropicBudgetTokens, ...overrides }),
+ ...buildAnthropicBodyWithReasoning(config, messages, { max_tokens: anthropicBudgetTokens, ...overrides }, streaming),
model,
}
},
parseStream: parseAnthropicLine,
+ parseResponse: parseAnthropicResponse,
+ streaming,
}
}
@@ -885,7 +932,9 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
// handles that plus any other path-illegal characters.
const encodedModel = encodeURIComponent(model)
return {
- url: `https://generativelanguage.googleapis.com/v1beta/models/${encodedModel}:streamGenerateContent?alt=sse`,
+ url: streaming
+ ? `https://generativelanguage.googleapis.com/v1beta/models/${encodedModel}:streamGenerateContent?alt=sse`
+ : `https://generativelanguage.googleapis.com/v1beta/models/${encodedModel}:generateContent`,
headers: mergeLlmRequestHeaders(config.customHeaders, {
"Content-Type": JSON_CONTENT_TYPE,
"x-goog-api-key": apiKey,
@@ -895,6 +944,8 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
reasoning: effectiveReasoning(config, overrides),
}),
parseStream: parseGoogleLine,
+ parseResponse: parseGoogleResponse,
+ streaming,
}
}
@@ -910,8 +961,10 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
"api-key": apiKey,
}),
buildBody: (messages, overrides) =>
- buildOpenAiCompatibleBody(config, messages, overrides),
+ buildOpenAiCompatibleBody(config, messages, overrides, streaming),
parseStream: parseOpenAiLine,
+ parseResponse: parseOpenAiResponse,
+ streaming,
}
}
@@ -933,10 +986,12 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
...localLlmOriginHeader(),
}),
buildBody: (messages, overrides) => ({
- ...buildOpenAiCompatibleBody(config, messages, overrides),
+ ...buildOpenAiCompatibleBody(config, messages, overrides, streaming),
model,
}),
parseStream: parseOpenAiLine,
+ parseResponse: parseOpenAiResponse,
+ streaming,
}
}
@@ -953,11 +1008,13 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
buildBody: (messages, overrides) => {
assertMiniMaxImageSupport(url, model, messages)
return {
- ...buildAnthropicBodyWithReasoning(config, messages, { max_tokens: anthropicBudgetTokens, ...overrides }),
+ ...buildAnthropicBodyWithReasoning(config, messages, { max_tokens: anthropicBudgetTokens, ...overrides }, streaming),
model,
}
},
parseStream: parseAnthropicLine,
+ parseResponse: parseAnthropicResponse,
+ streaming,
}
}
@@ -985,11 +1042,13 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
buildBody: (messages, overrides) => {
assertMiniMaxImageSupport(url, model, messages)
return {
- ...buildAnthropicBodyWithReasoning(config, messages, { max_tokens: anthropicBudgetTokens, ...overrides }),
+ ...buildAnthropicBodyWithReasoning(config, messages, { max_tokens: anthropicBudgetTokens, ...overrides }, streaming),
model,
}
},
parseStream: parseAnthropicLine,
+ parseResponse: parseAnthropicResponse,
+ streaming,
}
}
// Defense-in-depth: settings-side EndpointField normalizes URLs on
@@ -1023,11 +1082,13 @@ export function getProviderConfig(config: LlmConfig): ProviderConfig {
...(!azure && isLocalOrPrivateHttpEndpoint(url) ? localLlmOriginHeader() : {}),
}),
buildBody: (messages, overrides) => {
- const body = buildOpenAiCompatibleBody(config, messages, overrides)
+ const body = buildOpenAiCompatibleBody(config, messages, overrides, streaming)
if (!azure) body.model = model
return body
},
parseStream: parseOpenAiLine,
+ parseResponse: parseOpenAiResponse,
+ streaming,
}
}
diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts
index 8ad13c58..29da074d 100644
--- a/src/stores/wiki-store.ts
+++ b/src/stores/wiki-store.ts
@@ -45,6 +45,8 @@ interface LlmConfig {
codexCliTimeoutMinutes?: number
/** HTTP LLM request backstop. Defaults to 30 minutes for legacy configs. */
requestTimeoutMinutes?: number
+ /** Defaults to true. HTTP providers use a non-streaming wire when false. */
+ streamingEnabled?: boolean
/** Optional headers added to every HTTP request for this provider preset. */
customHeaders?: Record<string, string>
}
@@ -325,6 +327,7 @@ export interface ProviderOverride {
localCliIsolation?: boolean
codexCliTimeoutMinutes?: number
requestTimeoutMinutes?: number
+ streamingEnabled?: boolean
customHeaders?: Record<string, string>
}