YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(ai): keep Pipe rescue independent of flex switch (#5840)
Louis Beaumonte8ed10afed84
diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.tsindex 9a9b198ef..01c3c95fa 100644--- a/packages/ai-gateway/src/index.ts+++ b/packages/ai-gateway/src/index.ts@@ -94,6 +94,18 @@ import {export { RateLimiter };+/**+ * Keep paid background-Pipe rescue independent from the flex-tier kill switch.+ * The header identifies workload intent; resolveLatencyClass only controls+ * whether the primary provider may use flex capacity.+ */+export function shouldEnableArgusBackgroundFallback(+ request: Request,+ authResult: AuthResult,+): boolean {+ return isBackgroundRequest(request) && hasPaidHostedAiPlan(authResult);+}+type BoundedJsonRead =| { ok: true; value: unknown; bytes: number }| { ok: false; tooLarge: boolean };@@ -723,9 +735,23 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCofreePreview: freeChat.mode === 'metered',efficientOnly: getHostedAiPlan(authResult.accountPlan) !== 'business',gatewayContext,- argusBackgroundFallback: latency === 'background' && hasPaidHostedAiPlan(authResult),+ argusBackgroundFallback: shouldEnableArgusBackgroundFallback(request, authResult),},);+ if (response.status === 429 && body.stream) {+ const userAgent = request.headers.get('user-agent')?.toLowerCase() ?? '';+ console.warn('streaming hosted AI limit classification', {+ backgroundHeader: isBackgroundRequest(request),+ hasSessionAffinity: Boolean(+ request.headers.get('x-session-id')+ || request.headers.get('x-screenpipe-session-id'),+ ),+ piClient: userAgent.includes('pi-ai') || userAgent.includes('pi-coding-agent'),+ model: body.model,+ messageRoles: body.messages.map((message) => String(message.role)).join(',').slice(0, 256),+ toolCount: Array.isArray(body.tools) ? body.tools.length : 0,+ });+ }const latencyMs = Date.now() - reqStart;// Difficulty-router decision (null unless the router ran) for A/B measurement.const routerTier = response.headers.get('x-screenpipe-router-tier');diff --git a/packages/ai-gateway/src/providers/openai.ts b/packages/ai-gateway/src/providers/openai.tsindex 380e40172..b0d33a787 100644--- a/packages/ai-gateway/src/providers/openai.ts+++ b/packages/ai-gateway/src/providers/openai.ts@@ -340,6 +340,25 @@ export class OpenAIProvider implements AIProvider {this.client.chat.completions.create(p as ChatCompletionCreateParams & { stream: true }),)) as OpenAIChatStream;+ // OpenAI-compatible SDK streams are lazy: the HTTP request can succeed at+ // `create()` time and only surface an upstream 429 when the first chunk is+ // consumed. Prime that first chunk before returning a ReadableStream so the+ // gateway's outer model cascade can still recognize allowance exhaustion and+ // route paid background Pipes to Argus. Later mid-stream failures remain SSE+ // errors because a response may already have reached the client by then.+ const iterator = stream[Symbol.asyncIterator]();+ const firstChunk = await iterator.next();+ const primedStream = {+ async *[Symbol.asyncIterator]() {+ if (!firstChunk.done) yield firstChunk.value;+ while (true) {+ const next = await iterator.next();+ if (next.done) return;+ yield next.value;+ }+ },+ };+// Capture scope fields for the error path below — `this` inside the// ReadableStream start() refers to the controller, not the provider.const modelForTags = body.model;@@ -350,7 +369,7 @@ export class OpenAIProvider implements AIProvider {try {let finishReason: string | null = null;let usage: any = null;- for await (const chunk of stream) {+ for await (const chunk of primedStream) {// include_usage delivers a final chunk with empty choices// and the request's usage (incl. cached-token details)if ((chunk as any).usage) {diff --git a/packages/ai-gateway/src/services/background-limit-fallback.ts b/packages/ai-gateway/src/services/background-limit-fallback.tsindex 8d2bd72cb..e760a736b 100644--- a/packages/ai-gateway/src/services/background-limit-fallback.ts+++ b/packages/ai-gateway/src/services/background-limit-fallback.ts@@ -6,6 +6,7 @@ import type { Env, RequestBody } from '../types';import { isHostedChatAllowanceError } from './cloudflare-ai-gateway';export const ARGUS_BACKGROUND_FALLBACK_MODEL = 'argus-trace-1';+export const ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS = 2_048;const ARGUS_JSON_SYSTEM_PROMPT = 'Return only one valid JSON object matching the requested response format. Do not include markdown or prose.';@@ -50,10 +51,23 @@ export function prepareArgusBackgroundFallbackBody(body: RequestBody): RequestBoconst messages = body.response_format?.type === 'json_object' || body.response_format?.type === 'json_schema'? [{ role: 'system' as const, content: ARGUS_JSON_SYSTEM_PROMPT }, ...body.messages]: body.messages;+ const requestedTokens = body.max_completion_tokens ?? body.max_tokens;+ const maxTokens = Math.min(+ typeof requestedTokens === 'number' && Number.isFinite(requestedTokens) && requestedTokens > 0+ ? Math.floor(requestedTokens)+ : ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS,+ ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS,+ );return {...body,model: ARGUS_BACKGROUND_FALLBACK_MODEL,messages,+ // Pi advertises the primary hosted model's 32k output budget. Argus has an+ // 8,192-token total window and rejects that request before generating. Use+ // the broadly supported vLLM field and preserve most of the window for the+ // Pipe prompt and tool definitions.+ max_tokens: maxTokens,+ max_completion_tokens: undefined,};}diff --git a/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts b/packages/ai-gateway/src/test/background-argus-fallback.unit.test.tsindex 517d81b55..fef50177b 100644--- a/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts+++ b/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts@@ -6,6 +6,7 @@ import { describe, expect, it, mock } from 'bun:test';import { tryArgusBackgroundFallback } from '../handlers/chat';import {ARGUS_BACKGROUND_FALLBACK_MODEL,+ ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS,hasArgusUnsupportedInput,isAccountLocalAllowanceError,isProviderQuotaOrBillingLimitError,@@ -101,6 +102,19 @@ describe('paid background Pipe Argus fallback', () => {expect(textBody.messages).toBe(body.messages);});+ it('normalizes Pi\'s 32k output request to the bounded Argus vLLM field', () => {+ const capped = prepareArgusBackgroundFallbackBody({+ ...body,+ max_tokens: 16_000,+ max_completion_tokens: 32_000,+ });+ expect(capped.max_completion_tokens).toBeUndefined();+ expect(capped.max_tokens).toBe(ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS);++ const alreadyBounded = prepareArgusBackgroundFallbackBody({ ...body, max_tokens: 512 });+ expect(alreadyBounded.max_tokens).toBe(512);+ });+it('preserves the original allowance response when Argus is unavailable', async () => {const attempt = mock(async () => {throw Object.assign(new Error('Argus unavailable'), { status: 503 });diff --git a/packages/ai-gateway/src/test/free-chat-route.unit.test.ts b/packages/ai-gateway/src/test/free-chat-route.unit.test.tsindex e35cebe4a..425ef946e 100644--- a/packages/ai-gateway/src/test/free-chat-route.unit.test.ts+++ b/packages/ai-gateway/src/test/free-chat-route.unit.test.ts@@ -14,7 +14,8 @@ mock.module('@clerk/backend', () => ({verifyToken: verifyTokenMock,}));-const { handleRequest } = await import('../index');+const { handleRequest, shouldEnableArgusBackgroundFallback } = await import('../index');+const { resolveLatencyClass } = await import('../utils/latency');describe('/v1/chat/completions free-plan route policy', () => {const originalFetch = globalThis.fetch;@@ -159,6 +160,29 @@ describe('/v1/chat/completions free-plan route policy', () => {expect(await response.text()).not.toContain('authentication_required');});+ it('keeps the paid background Argus rescue enabled when the flex-tier kill switch is off', () => {+ const backgroundRequest = request({+ 'x-screenpipe-latency': 'background',+ });+ const flexDisabledEnv = {+ ...env,+ FLEX_TIER_ENABLED: 'false',+ } as unknown as Env;++ // The flex switch may force standard provider latency, but it must not+ // turn a paid Pipe into interactive traffic for allowance rescue.+ expect(resolveLatencyClass(backgroundRequest, {+ model: 'auto',+ messages: [{ role: 'user', content: 'hello' }],+ }, flexDisabledEnv)).toBe('interactive');+ expect(shouldEnableArgusBackgroundFallback(backgroundRequest, {+ isValid: true,+ tier: 'subscribed',+ accountPlan: 'basic',+ deviceId: 'pipe-device',+ })).toBe(true);+ });+it('blocks a free hosted background request instead of trusting its header', async () => {const authFetch = mock(async (input: RequestInfo | URL) => {expect(String(input)).toBe('https://screenpipe.com/api/user');diff --git a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.tsindex 7c7987a96..0ced3d989 100644--- a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts+++ b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts@@ -51,6 +51,21 @@ const body: RequestBody = {};describe('OpenAIProvider streaming — tool calls', () => {+ it('rejects a first-chunk upstream error so the gateway can run its fallback', async () => {+ async function* stream() {+ throw Object.assign(new Error('provider quota or billing limit reached'), {+ status: 429,+ code: 'insufficient_quota',+ });+ }++ const provider = makeOpenAIProvider(stream);+ await expect(provider.createStreamingCompletion(body)).rejects.toMatchObject({+ status: 429,+ code: 'insufficient_quota',+ });+ });+it('forwards streamed tool_calls deltas (name + accumulated arguments + finish_reason)', async () => {// Mirrors how OpenAI streams a tool call: name arrives first, then the// JSON arguments arrive fragmented across chunks.