Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai-gateway): make GLM Pi responses visible and fast (#6891)

Louis Beaumont3a5fc7eec382
diff --git a/packages/ai-gateway/src/handlers/models.ts b/packages/ai-gateway/src/handlers/models.ts
index 482eb7a27..fdfed2689 100644
--- a/packages/ai-gateway/src/handlers/models.ts
+++ b/packages/ai-gateway/src/handlers/models.ts
@@ -313,7 +313,7 @@ const CURATED_MODELS: ModelEntry[] = [
intelligence: 'high',
cost_tier: 'low',
recommended_for: ['chat', 'analysis'],
- warning: 'Experimental single-slot model — requests may queue under load; no vision or tool calling',
+ warning: 'Experimental single-slot, text-only model — requests may queue under load',
requires_env: 'TINFOIL_GLM_API_KEY',
},
];
diff --git a/packages/ai-gateway/src/providers/openai.ts b/packages/ai-gateway/src/providers/openai.ts
index 9eaf70a97..f3b530bae 100644
--- a/packages/ai-gateway/src/providers/openai.ts
+++ b/packages/ai-gateway/src/providers/openai.ts
@@ -246,9 +246,14 @@ export class OpenAIProvider implements AIProvider {
this.chatTemplateKwargs = chatTemplateKwargs;
}
- private applyProviderOptions(params: ChatCompletionCreateParams): void {
- if (this.chatTemplateKwargs) {
- Object.assign(params, { chat_template_kwargs: this.chatTemplateKwargs });
+ protected getChatTemplateKwargs(_body: RequestBody): Record<string, unknown> | undefined {
+ return this.chatTemplateKwargs;
+ }
+
+ private applyProviderOptions(params: ChatCompletionCreateParams, body: RequestBody): void {
+ const chatTemplateKwargs = this.getChatTemplateKwargs(body);
+ if (chatTemplateKwargs) {
+ Object.assign(params, { chat_template_kwargs: chatTemplateKwargs });
}
}
@@ -399,7 +404,7 @@ export class OpenAIProvider implements AIProvider {
this.applyGenerationOptions(params, body);
this.applyTokenLimit(params, body);
this.applyToolCompatibilityOptions(params, body);
- this.applyProviderOptions(params);
+ this.applyProviderOptions(params, body);
await applyGpt56PromptCaching(params, body.gpt56HistoryCacheEligible === true);
const response = await this.createWithUnsupportedParamRetry(params, (p) =>
@@ -433,7 +438,7 @@ export class OpenAIProvider implements AIProvider {
this.applyGenerationOptions(params, body);
this.applyTokenLimit(params, body);
this.applyToolCompatibilityOptions(params, body);
- this.applyProviderOptions(params);
+ this.applyProviderOptions(params, body);
await applyGpt56PromptCaching(params, body.gpt56HistoryCacheEligible === true);
const stream = (await this.createWithUnsupportedParamRetry(params, (p) =>
@@ -497,6 +502,24 @@ export class OpenAIProvider implements AIProvider {
)
);
}
+ // OpenAI-compatible reasoning models (notably llama.cpp GLM)
+ // stream their work before final text. Preserve the first
+ // supported reasoning field so Pi receives thinking deltas and
+ // the UI can show liveness instead of appearing frozen. This is
+ // also the only model output when generation ends inside the
+ // reasoning budget, so dropping it produced an empty response.
+ const delta = choice?.delta as Record<string, unknown> | undefined;
+ const reasoningField = ['reasoning_content', 'reasoning', 'reasoning_text']
+ .find((field) => typeof delta?.[field] === 'string' && (delta[field] as string).length > 0);
+ if (reasoningField && delta) {
+ controller.enqueue(
+ new TextEncoder().encode(
+ `data: ${JSON.stringify({
+ choices: [{ delta: { [reasoningField]: delta[reasoningField] } }],
+ })}\n\n`
+ )
+ );
+ }
if (body.response_format?.type === 'json_object' || body.response_format?.type === 'json_schema') {
const content = choice?.delta?.content;
if (content) {
diff --git a/packages/ai-gateway/src/providers/screenpipe-glm.ts b/packages/ai-gateway/src/providers/screenpipe-glm.ts
index 3da8cb2a7..e4cbeb905 100644
--- a/packages/ai-gateway/src/providers/screenpipe-glm.ts
+++ b/packages/ai-gateway/src/providers/screenpipe-glm.ts
@@ -168,6 +168,16 @@ function normalizeGlmToolCallStream(stream: ReadableStream, tools: unknown[]): R
emit(raw);
return;
}
+ // Reasoning arrives before GLM's final content. It cannot be a
+ // native tool-call payload, so forward it immediately while the
+ // content-only detector remains undecided. Buffering these chunks
+ // made long-thinking requests look completely frozen in Pi.
+ const hasReasoning = ['reasoning_content', 'reasoning', 'reasoning_text']
+ .some((field) => typeof choice?.delta?.[field] === 'string' && choice.delta[field].length > 0);
+ if (hasReasoning && typeof deltaContent !== 'string') {
+ emit(raw);
+ return;
+ }
pending.push(raw);
if (typeof deltaContent === 'string') {
@@ -253,6 +263,18 @@ export class ScreenpipeGlmProvider extends OpenAIProvider {
super(apiKey, baseURL, defaultHeaders, maxRetries);
}
+ protected getChatTemplateKwargs(body: RequestBody): Record<string, unknown> {
+ // The deployed GLM template has binary thinking, not graded effort.
+ // Pi defaults interactive chats to "medium" and its 32K Day Recap
+ // prompt already consumes most of that window. Treat medium and below
+ // as the fast/non-thinking path; high+ remains an explicit opt-in.
+ return {
+ enable_thinking: body.reasoning_effort === 'high'
+ || body.reasoning_effort === 'xhigh'
+ || String(body.reasoning_effort) === 'max',
+ };
+ }
+
async createCompletion(body: RequestBody): Promise<Response> {
const response = await super.createCompletion(normalizeGlmRequest(body));
if (!Array.isArray(body.tools) || body.tools.length === 0) return response;
diff --git a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts
index dcb8937a6..b07a58e8b 100644
--- a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts
+++ b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts
@@ -199,4 +199,91 @@ describe('OpenAIProvider streaming — tool calls', () => {
expect(JSON.parse(toolCall.function.arguments)).toEqual({ path: '/tmp/pi-tool-test' });
expect(events.map((event) => event.choices?.[0]?.finish_reason).find(Boolean)).toBe('tool_calls');
});
+
+ it('forwards GLM reasoning immediately while native tool content remains buffered', async () => {
+ let releaseContent!: () => void;
+ const contentGate = new Promise<void>((resolve) => {
+ releaseContent = resolve;
+ });
+ async function* stream() {
+ yield { choices: [{ delta: { reasoning_content: 'I should inspect the file.' } }] };
+ await contentGate;
+ yield { choices: [{ delta: { content: '<tool_call>read' } }] };
+ yield { choices: [{ delta: { content: '{"path":"/tmp/pi-tool-test"}' } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+ }
+
+ const provider = makeGlmProvider(stream);
+ const glmBody: RequestBody = {
+ model: 'glm-5.3-flash-reap50-iq3m',
+ reasoning_effort: 'high',
+ messages: [{ role: 'user', content: 'Read the test file.' }],
+ tools: [{
+ type: 'function',
+ function: {
+ name: 'read',
+ description: 'Read a local file',
+ parameters: { type: 'object', properties: { path: { type: 'string' } } },
+ },
+ }],
+ };
+ const reader = (await provider.createStreamingCompletion(glmBody)).getReader();
+ const first = await Promise.race([
+ reader.read().then((value) => ({ kind: 'chunk' as const, value })),
+ new Promise<{ kind: 'timeout' }>((resolve) => setTimeout(() => resolve({ kind: 'timeout' }), 250)),
+ ]);
+
+ expect(first.kind).toBe('chunk');
+ if (first.kind !== 'chunk') throw new Error('reasoning delta was buffered');
+ const firstText = new TextDecoder().decode(first.value.value);
+ expect(parseEvents(firstText)[0].choices[0].delta.reasoning_content).toBe('I should inspect the file.');
+
+ releaseContent();
+ let remaining = '';
+ while (true) {
+ const next = await reader.read();
+ if (next.done) break;
+ remaining += new TextDecoder().decode(next.value);
+ }
+ const remainingEvents = parseEvents(remaining);
+ expect(remainingEvents.map((event) => event.choices?.[0]?.delta?.content ?? '').join('')).toBe('');
+ expect(remainingEvents.flatMap((event) => event.choices?.[0]?.delta?.tool_calls ?? [])[0].function.name).toBe('read');
+ });
+
+ it('uses fast GLM template mode by default and keeps high reasoning as opt-in', async () => {
+ const requests: any[] = [];
+ const provider = new ScreenpipeGlmProvider('test-key');
+ (provider as any).client = {
+ baseURL: 'https://pii.screenpipe.containers.tinfoil.dev/glm/v1',
+ chat: {
+ completions: {
+ create: async (params: any) => {
+ requests.push(params);
+ const response = (async function* () {
+ yield { choices: [{ delta: { content: 'ok' } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+ })();
+ (response as any).controller = { abort: () => {} };
+ return response;
+ },
+ },
+ },
+ };
+
+ for (const reasoning_effort of [undefined, 'medium', 'high', 'max'] as const) {
+ const request: RequestBody = {
+ model: 'glm-5.3-flash-reap50-iq3m',
+ messages: [{ role: 'user', content: 'hello' }],
+ ...(reasoning_effort ? { reasoning_effort: reasoning_effort as RequestBody['reasoning_effort'] } : {}),
+ };
+ await new Response(await provider.createStreamingCompletion(request)).text();
+ }
+
+ expect(requests.map((request) => request.chat_template_kwargs?.enable_thinking)).toEqual([
+ false,
+ false,
+ true,
+ true,
+ ]);
+ });
});