YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(ai-gateway): make GLM Pi responses visible and fast (#6891)
Louis Beaumont3a5fc7eec382
diff --git a/packages/ai-gateway/src/handlers/models.ts b/packages/ai-gateway/src/handlers/models.tsindex 482eb7a27..fdfed2689 100644--- a/packages/ai-gateway/src/handlers/models.ts+++ b/packages/ai-gateway/src/handlers/models.ts@@ -313,7 +313,7 @@ const CURATED_MODELS: ModelEntry[] = [intelligence: 'high',cost_tier: 'low',recommended_for: ['chat', 'analysis'],- warning: 'Experimental single-slot model — requests may queue under load; no vision or tool calling',+ warning: 'Experimental single-slot, text-only model — requests may queue under load',requires_env: 'TINFOIL_GLM_API_KEY',},];diff --git a/packages/ai-gateway/src/providers/openai.ts b/packages/ai-gateway/src/providers/openai.tsindex 9eaf70a97..f3b530bae 100644--- a/packages/ai-gateway/src/providers/openai.ts+++ b/packages/ai-gateway/src/providers/openai.ts@@ -246,9 +246,14 @@ export class OpenAIProvider implements AIProvider {this.chatTemplateKwargs = chatTemplateKwargs;}- private applyProviderOptions(params: ChatCompletionCreateParams): void {- if (this.chatTemplateKwargs) {- Object.assign(params, { chat_template_kwargs: this.chatTemplateKwargs });+ protected getChatTemplateKwargs(_body: RequestBody): Record<string, unknown> | undefined {+ return this.chatTemplateKwargs;+ }++ private applyProviderOptions(params: ChatCompletionCreateParams, body: RequestBody): void {+ const chatTemplateKwargs = this.getChatTemplateKwargs(body);+ if (chatTemplateKwargs) {+ Object.assign(params, { chat_template_kwargs: chatTemplateKwargs });}}@@ -399,7 +404,7 @@ export class OpenAIProvider implements AIProvider {this.applyGenerationOptions(params, body);this.applyTokenLimit(params, body);this.applyToolCompatibilityOptions(params, body);- this.applyProviderOptions(params);+ this.applyProviderOptions(params, body);await applyGpt56PromptCaching(params, body.gpt56HistoryCacheEligible === true);const response = await this.createWithUnsupportedParamRetry(params, (p) =>@@ -433,7 +438,7 @@ export class OpenAIProvider implements AIProvider {this.applyGenerationOptions(params, body);this.applyTokenLimit(params, body);this.applyToolCompatibilityOptions(params, body);- this.applyProviderOptions(params);+ this.applyProviderOptions(params, body);await applyGpt56PromptCaching(params, body.gpt56HistoryCacheEligible === true);const stream = (await this.createWithUnsupportedParamRetry(params, (p) =>@@ -497,6 +502,24 @@ export class OpenAIProvider implements AIProvider {));}+ // OpenAI-compatible reasoning models (notably llama.cpp GLM)+ // stream their work before final text. Preserve the first+ // supported reasoning field so Pi receives thinking deltas and+ // the UI can show liveness instead of appearing frozen. This is+ // also the only model output when generation ends inside the+ // reasoning budget, so dropping it produced an empty response.+ const delta = choice?.delta as Record<string, unknown> | undefined;+ const reasoningField = ['reasoning_content', 'reasoning', 'reasoning_text']+ .find((field) => typeof delta?.[field] === 'string' && (delta[field] as string).length > 0);+ if (reasoningField && delta) {+ controller.enqueue(+ new TextEncoder().encode(+ `data: ${JSON.stringify({+ choices: [{ delta: { [reasoningField]: delta[reasoningField] } }],+ })}\n\n`+ )+ );+ }if (body.response_format?.type === 'json_object' || body.response_format?.type === 'json_schema') {const content = choice?.delta?.content;if (content) {diff --git a/packages/ai-gateway/src/providers/screenpipe-glm.ts b/packages/ai-gateway/src/providers/screenpipe-glm.tsindex 3da8cb2a7..e4cbeb905 100644--- a/packages/ai-gateway/src/providers/screenpipe-glm.ts+++ b/packages/ai-gateway/src/providers/screenpipe-glm.ts@@ -168,6 +168,16 @@ function normalizeGlmToolCallStream(stream: ReadableStream, tools: unknown[]): Remit(raw);return;}+ // Reasoning arrives before GLM's final content. It cannot be a+ // native tool-call payload, so forward it immediately while the+ // content-only detector remains undecided. Buffering these chunks+ // made long-thinking requests look completely frozen in Pi.+ const hasReasoning = ['reasoning_content', 'reasoning', 'reasoning_text']+ .some((field) => typeof choice?.delta?.[field] === 'string' && choice.delta[field].length > 0);+ if (hasReasoning && typeof deltaContent !== 'string') {+ emit(raw);+ return;+ }pending.push(raw);if (typeof deltaContent === 'string') {@@ -253,6 +263,18 @@ export class ScreenpipeGlmProvider extends OpenAIProvider {super(apiKey, baseURL, defaultHeaders, maxRetries);}+ protected getChatTemplateKwargs(body: RequestBody): Record<string, unknown> {+ // The deployed GLM template has binary thinking, not graded effort.+ // Pi defaults interactive chats to "medium" and its 32K Day Recap+ // prompt already consumes most of that window. Treat medium and below+ // as the fast/non-thinking path; high+ remains an explicit opt-in.+ return {+ enable_thinking: body.reasoning_effort === 'high'+ || body.reasoning_effort === 'xhigh'+ || String(body.reasoning_effort) === 'max',+ };+ }+async createCompletion(body: RequestBody): Promise<Response> {const response = await super.createCompletion(normalizeGlmRequest(body));if (!Array.isArray(body.tools) || body.tools.length === 0) return response;diff --git a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.tsindex dcb8937a6..b07a58e8b 100644--- a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts+++ b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts@@ -199,4 +199,91 @@ describe('OpenAIProvider streaming — tool calls', () => {expect(JSON.parse(toolCall.function.arguments)).toEqual({ path: '/tmp/pi-tool-test' });expect(events.map((event) => event.choices?.[0]?.finish_reason).find(Boolean)).toBe('tool_calls');});++ it('forwards GLM reasoning immediately while native tool content remains buffered', async () => {+ let releaseContent!: () => void;+ const contentGate = new Promise<void>((resolve) => {+ releaseContent = resolve;+ });+ async function* stream() {+ yield { choices: [{ delta: { reasoning_content: 'I should inspect the file.' } }] };+ await contentGate;+ yield { choices: [{ delta: { content: '<tool_call>read' } }] };+ yield { choices: [{ delta: { content: '{"path":"/tmp/pi-tool-test"}' } }] };+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ }++ const provider = makeGlmProvider(stream);+ const glmBody: RequestBody = {+ model: 'glm-5.3-flash-reap50-iq3m',+ reasoning_effort: 'high',+ messages: [{ role: 'user', content: 'Read the test file.' }],+ tools: [{+ type: 'function',+ function: {+ name: 'read',+ description: 'Read a local file',+ parameters: { type: 'object', properties: { path: { type: 'string' } } },+ },+ }],+ };+ const reader = (await provider.createStreamingCompletion(glmBody)).getReader();+ const first = await Promise.race([+ reader.read().then((value) => ({ kind: 'chunk' as const, value })),+ new Promise<{ kind: 'timeout' }>((resolve) => setTimeout(() => resolve({ kind: 'timeout' }), 250)),+ ]);++ expect(first.kind).toBe('chunk');+ if (first.kind !== 'chunk') throw new Error('reasoning delta was buffered');+ const firstText = new TextDecoder().decode(first.value.value);+ expect(parseEvents(firstText)[0].choices[0].delta.reasoning_content).toBe('I should inspect the file.');++ releaseContent();+ let remaining = '';+ while (true) {+ const next = await reader.read();+ if (next.done) break;+ remaining += new TextDecoder().decode(next.value);+ }+ const remainingEvents = parseEvents(remaining);+ expect(remainingEvents.map((event) => event.choices?.[0]?.delta?.content ?? '').join('')).toBe('');+ expect(remainingEvents.flatMap((event) => event.choices?.[0]?.delta?.tool_calls ?? [])[0].function.name).toBe('read');+ });++ it('uses fast GLM template mode by default and keeps high reasoning as opt-in', async () => {+ const requests: any[] = [];+ const provider = new ScreenpipeGlmProvider('test-key');+ (provider as any).client = {+ baseURL: 'https://pii.screenpipe.containers.tinfoil.dev/glm/v1',+ chat: {+ completions: {+ create: async (params: any) => {+ requests.push(params);+ const response = (async function* () {+ yield { choices: [{ delta: { content: 'ok' } }] };+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ })();+ (response as any).controller = { abort: () => {} };+ return response;+ },+ },+ },+ };++ for (const reasoning_effort of [undefined, 'medium', 'high', 'max'] as const) {+ const request: RequestBody = {+ model: 'glm-5.3-flash-reap50-iq3m',+ messages: [{ role: 'user', content: 'hello' }],+ ...(reasoning_effort ? { reasoning_effort: reasoning_effort as RequestBody['reasoning_effort'] } : {}),+ };+ await new Response(await provider.createStreamingCompletion(request)).text();+ }++ expect(requests.map((request) => request.chat_template_kwargs?.enable_thinking)).toEqual([+ false,+ false,+ true,+ true,+ ]);+ });});