Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai): keep Pipe rescue independent of flex switch (#5840)

Louis Beaumonte8ed10afed84
diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.ts
index 9a9b198ef..01c3c95fa 100644
--- a/packages/ai-gateway/src/index.ts
+++ b/packages/ai-gateway/src/index.ts
@@ -94,6 +94,18 @@ import {
export { RateLimiter };
+/**
+ * Keep paid background-Pipe rescue independent from the flex-tier kill switch.
+ * The header identifies workload intent; resolveLatencyClass only controls
+ * whether the primary provider may use flex capacity.
+ */
+export function shouldEnableArgusBackgroundFallback(
+ request: Request,
+ authResult: AuthResult,
+): boolean {
+ return isBackgroundRequest(request) && hasPaidHostedAiPlan(authResult);
+}
+
type BoundedJsonRead =
| { ok: true; value: unknown; bytes: number }
| { ok: false; tooLarge: boolean };
@@ -723,9 +735,23 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
freePreview: freeChat.mode === 'metered',
efficientOnly: getHostedAiPlan(authResult.accountPlan) !== 'business',
gatewayContext,
- argusBackgroundFallback: latency === 'background' && hasPaidHostedAiPlan(authResult),
+ argusBackgroundFallback: shouldEnableArgusBackgroundFallback(request, authResult),
},
);
+ if (response.status === 429 && body.stream) {
+ const userAgent = request.headers.get('user-agent')?.toLowerCase() ?? '';
+ console.warn('streaming hosted AI limit classification', {
+ backgroundHeader: isBackgroundRequest(request),
+ hasSessionAffinity: Boolean(
+ request.headers.get('x-session-id')
+ || request.headers.get('x-screenpipe-session-id'),
+ ),
+ piClient: userAgent.includes('pi-ai') || userAgent.includes('pi-coding-agent'),
+ model: body.model,
+ messageRoles: body.messages.map((message) => String(message.role)).join(',').slice(0, 256),
+ toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
+ });
+ }
const latencyMs = Date.now() - reqStart;
// Difficulty-router decision (null unless the router ran) for A/B measurement.
const routerTier = response.headers.get('x-screenpipe-router-tier');
diff --git a/packages/ai-gateway/src/providers/openai.ts b/packages/ai-gateway/src/providers/openai.ts
index 380e40172..b0d33a787 100644
--- a/packages/ai-gateway/src/providers/openai.ts
+++ b/packages/ai-gateway/src/providers/openai.ts
@@ -340,6 +340,25 @@ export class OpenAIProvider implements AIProvider {
this.client.chat.completions.create(p as ChatCompletionCreateParams & { stream: true }),
)) as OpenAIChatStream;
+ // OpenAI-compatible SDK streams are lazy: the HTTP request can succeed at
+ // `create()` time and only surface an upstream 429 when the first chunk is
+ // consumed. Prime that first chunk before returning a ReadableStream so the
+ // gateway's outer model cascade can still recognize allowance exhaustion and
+ // route paid background Pipes to Argus. Later mid-stream failures remain SSE
+ // errors because a response may already have reached the client by then.
+ const iterator = stream[Symbol.asyncIterator]();
+ const firstChunk = await iterator.next();
+ const primedStream = {
+ async *[Symbol.asyncIterator]() {
+ if (!firstChunk.done) yield firstChunk.value;
+ while (true) {
+ const next = await iterator.next();
+ if (next.done) return;
+ yield next.value;
+ }
+ },
+ };
+
// Capture scope fields for the error path below — `this` inside the
// ReadableStream start() refers to the controller, not the provider.
const modelForTags = body.model;
@@ -350,7 +369,7 @@ export class OpenAIProvider implements AIProvider {
try {
let finishReason: string | null = null;
let usage: any = null;
- for await (const chunk of stream) {
+ for await (const chunk of primedStream) {
// include_usage delivers a final chunk with empty choices
// and the request's usage (incl. cached-token details)
if ((chunk as any).usage) {
diff --git a/packages/ai-gateway/src/services/background-limit-fallback.ts b/packages/ai-gateway/src/services/background-limit-fallback.ts
index 8d2bd72cb..e760a736b 100644
--- a/packages/ai-gateway/src/services/background-limit-fallback.ts
+++ b/packages/ai-gateway/src/services/background-limit-fallback.ts
@@ -6,6 +6,7 @@ import type { Env, RequestBody } from '../types';
import { isHostedChatAllowanceError } from './cloudflare-ai-gateway';
export const ARGUS_BACKGROUND_FALLBACK_MODEL = 'argus-trace-1';
+export const ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS = 2_048;
const ARGUS_JSON_SYSTEM_PROMPT = 'Return only one valid JSON object matching the requested response format. Do not include markdown or prose.';
@@ -50,10 +51,23 @@ export function prepareArgusBackgroundFallbackBody(body: RequestBody): RequestBo
const messages = body.response_format?.type === 'json_object' || body.response_format?.type === 'json_schema'
? [{ role: 'system' as const, content: ARGUS_JSON_SYSTEM_PROMPT }, ...body.messages]
: body.messages;
+ const requestedTokens = body.max_completion_tokens ?? body.max_tokens;
+ const maxTokens = Math.min(
+ typeof requestedTokens === 'number' && Number.isFinite(requestedTokens) && requestedTokens > 0
+ ? Math.floor(requestedTokens)
+ : ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS,
+ ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS,
+ );
return {
...body,
model: ARGUS_BACKGROUND_FALLBACK_MODEL,
messages,
+ // Pi advertises the primary hosted model's 32k output budget. Argus has an
+ // 8,192-token total window and rejects that request before generating. Use
+ // the broadly supported vLLM field and preserve most of the window for the
+ // Pipe prompt and tool definitions.
+ max_tokens: maxTokens,
+ max_completion_tokens: undefined,
};
}
diff --git a/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts b/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts
index 517d81b55..fef50177b 100644
--- a/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts
+++ b/packages/ai-gateway/src/test/background-argus-fallback.unit.test.ts
@@ -6,6 +6,7 @@ import { describe, expect, it, mock } from 'bun:test';
import { tryArgusBackgroundFallback } from '../handlers/chat';
import {
ARGUS_BACKGROUND_FALLBACK_MODEL,
+ ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS,
hasArgusUnsupportedInput,
isAccountLocalAllowanceError,
isProviderQuotaOrBillingLimitError,
@@ -101,6 +102,19 @@ describe('paid background Pipe Argus fallback', () => {
expect(textBody.messages).toBe(body.messages);
});
+ it('normalizes Pi\'s 32k output request to the bounded Argus vLLM field', () => {
+ const capped = prepareArgusBackgroundFallbackBody({
+ ...body,
+ max_tokens: 16_000,
+ max_completion_tokens: 32_000,
+ });
+ expect(capped.max_completion_tokens).toBeUndefined();
+ expect(capped.max_tokens).toBe(ARGUS_BACKGROUND_MAX_COMPLETION_TOKENS);
+
+ const alreadyBounded = prepareArgusBackgroundFallbackBody({ ...body, max_tokens: 512 });
+ expect(alreadyBounded.max_tokens).toBe(512);
+ });
+
it('preserves the original allowance response when Argus is unavailable', async () => {
const attempt = mock(async () => {
throw Object.assign(new Error('Argus unavailable'), { status: 503 });
diff --git a/packages/ai-gateway/src/test/free-chat-route.unit.test.ts b/packages/ai-gateway/src/test/free-chat-route.unit.test.ts
index e35cebe4a..425ef946e 100644
--- a/packages/ai-gateway/src/test/free-chat-route.unit.test.ts
+++ b/packages/ai-gateway/src/test/free-chat-route.unit.test.ts
@@ -14,7 +14,8 @@ mock.module('@clerk/backend', () => ({
verifyToken: verifyTokenMock,
}));
-const { handleRequest } = await import('../index');
+const { handleRequest, shouldEnableArgusBackgroundFallback } = await import('../index');
+const { resolveLatencyClass } = await import('../utils/latency');
describe('/v1/chat/completions free-plan route policy', () => {
const originalFetch = globalThis.fetch;
@@ -159,6 +160,29 @@ describe('/v1/chat/completions free-plan route policy', () => {
expect(await response.text()).not.toContain('authentication_required');
});
+ it('keeps the paid background Argus rescue enabled when the flex-tier kill switch is off', () => {
+ const backgroundRequest = request({
+ 'x-screenpipe-latency': 'background',
+ });
+ const flexDisabledEnv = {
+ ...env,
+ FLEX_TIER_ENABLED: 'false',
+ } as unknown as Env;
+
+ // The flex switch may force standard provider latency, but it must not
+ // turn a paid Pipe into interactive traffic for allowance rescue.
+ expect(resolveLatencyClass(backgroundRequest, {
+ model: 'auto',
+ messages: [{ role: 'user', content: 'hello' }],
+ }, flexDisabledEnv)).toBe('interactive');
+ expect(shouldEnableArgusBackgroundFallback(backgroundRequest, {
+ isValid: true,
+ tier: 'subscribed',
+ accountPlan: 'basic',
+ deviceId: 'pipe-device',
+ })).toBe(true);
+ });
+
it('blocks a free hosted background request instead of trusting its header', async () => {
const authFetch = mock(async (input: RequestInfo | URL) => {
expect(String(input)).toBe('https://screenpipe.com/api/user');
diff --git a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts
index 7c7987a96..0ced3d989 100644
--- a/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts
+++ b/packages/ai-gateway/src/test/openai-streaming-tool-calls.unit.test.ts
@@ -51,6 +51,21 @@ const body: RequestBody = {
};
describe('OpenAIProvider streaming — tool calls', () => {
+ it('rejects a first-chunk upstream error so the gateway can run its fallback', async () => {
+ async function* stream() {
+ throw Object.assign(new Error('provider quota or billing limit reached'), {
+ status: 429,
+ code: 'insufficient_quota',
+ });
+ }
+
+ const provider = makeOpenAIProvider(stream);
+ await expect(provider.createStreamingCompletion(body)).rejects.toMatchObject({
+ status: 429,
+ code: 'insufficient_quota',
+ });
+ });
+
it('forwards streamed tool_calls deltas (name + accumulated arguments + finish_reason)', async () => {
// Mirrors how OpenAI streams a tool call: name arrives first, then the
// JSON arguments arrive fragmented across chunks.