Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai-gateway): recover allowances on Luna and retain cost telemetry (#7386)

Louis Beaumont7a320188f551
diff --git a/packages/ai-gateway-tail/src/index.test.ts b/packages/ai-gateway-tail/src/index.test.ts
index 5bc9cd805..92e3fb256 100644
--- a/packages/ai-gateway-tail/src/index.test.ts
+++ b/packages/ai-gateway-tail/src/index.test.ts
@@ -27,6 +27,16 @@ function tailItem() {
}
describe('AI gateway Tail Worker', () => {
+ it('retains direct rescue attribution without forwarding arbitrary failure text', () => {
+ const item = tailItem();
+ item.event.response.status = 200;
+ item.logs = [{ message: [`screenpipe.ai-gateway-route ${JSON.stringify({
+ served_model: 'gpt-6-luna', gateway_mode: 'direct', fallback_model: 'gpt-6-luna', fallback_reason: 'account_allowance',
+ })}`] }];
+ expect(normalizeTailItem(item)).toMatchObject({ outcome: 'success', gateway_mode: 'direct', fallback_model: 'gpt-6-luna', fallback_reason: 'account_allowance' });
+ item.logs = [{ message: [`screenpipe.ai-gateway-route ${JSON.stringify({ fallback_reason: 'private provider payload' })}`] }];
+ expect(normalizeTailItem(item)?.fallback_reason).toBeUndefined();
+ });
it('exports a strict content-free request audit event', () => {
const event = normalizeTailItem(tailItem());
expect(event).toMatchObject({
diff --git a/packages/ai-gateway-tail/src/index.ts b/packages/ai-gateway-tail/src/index.ts
index 4a750c9fd..a057fe0d4 100644
--- a/packages/ai-gateway-tail/src/index.ts
+++ b/packages/ai-gateway-tail/src/index.ts
@@ -69,6 +69,8 @@ export interface AiGatewayAuditEvent {
router_tier?: string;
workload?: string;
gateway_mode?: string;
+ fallback_model?: string;
+ fallback_reason?: string;
latency_ms?: number;
exception_type?: string;
}
@@ -197,6 +199,9 @@ export function normalizeTailItem(item: TailItem): AiGatewayAuditEvent | null {
router_tier: token(route.router_tier, 24),
workload: token(route.workload, 24),
gateway_mode: token(route.gateway_mode, 24),
+ fallback_model: model(route.fallback_model),
+ fallback_reason: ['account_allowance', 'safety_refusal', 'allowance_or_quota'].includes(String(route.fallback_reason))
+ ? String(route.fallback_reason) : undefined,
latency_ms: latency,
exception_type: token(firstException?.name, 80),
};
diff --git a/packages/ai-gateway/src/handlers/chat.ts b/packages/ai-gateway/src/handlers/chat.ts
index 8f81fdceb..56de3ef44 100644
--- a/packages/ai-gateway/src/handlers/chat.ts
+++ b/packages/ai-gateway/src/handlers/chat.ts
@@ -22,7 +22,6 @@ import {
} from '../services/cloudflare-ai-gateway';
import { classifyCloudflareSpendLimitRule } from '../services/cloudflare-ai-gateway-usage';
import {
- BACKGROUND_FALLBACK_MODEL,
SafetyRefusalError,
isProviderQuotaOrBillingLimitError,
isSafetyRefusalError,
@@ -328,7 +327,8 @@ async function tryModel(
lane: error.allowance.lane,
limitScope: error.allowance.limit_scope,
});
- logModelOutcome(env, { model, outcome: 'rate_limited' }).catch(() => {});
+ // Account policy rejected the request before the provider ran. This is
+ // observable above, but is not a provider-health failure.
throw error;
}
@@ -535,7 +535,7 @@ export async function tryBackgroundFallback(
// No gateway context: call the provider directly so the exhausted
// Cloudflare spend limit cannot reject the rescue attempt too.
const response = await attemptModel(
- BACKGROUND_FALLBACK_MODEL,
+ fallbackBody.model,
fallbackBody,
env,
'fallback',
@@ -544,11 +544,14 @@ export async function tryBackgroundFallback(
);
console.warn('background hosted AI request served by rescue model', {
requestedModel: body.model,
- fallbackModel: BACKGROUND_FALLBACK_MODEL,
+ fallbackModel: fallbackBody.model,
reason: isSafetyRefusalError(error) ? 'safety_refusal' : 'allowance_or_quota',
});
- const tagged = addModelHeader(response, BACKGROUND_FALLBACK_MODEL);
- tagged.headers.set('x-screenpipe-background-fallback', BACKGROUND_FALLBACK_MODEL);
+ const tagged = addModelHeader(response, fallbackBody.model);
+ tagged.headers.set('x-screenpipe-background-fallback', fallbackBody.model);
+ tagged.headers.set('x-screenpipe-background-fallback-reason',
+ isHostedChatAllowanceError(error) ? 'account_allowance'
+ : isSafetyRefusalError(error) ? 'safety_refusal' : 'allowance_or_quota');
return addCorsHeaders(tagged);
} catch (fallbackError: any) {
console.error('background rescue fallback failed; preserving original response', {
@@ -812,7 +815,7 @@ function allowanceMessage(canUpgrade: boolean): string {
/** Render the stable terminal contract Pi uses to avoid generic 429 retries. */
export function allowanceErrorResponse(body: RequestBody, error: HostedChatAllowanceExceededError): Response {
- const upgrade = error.allowance.plan === 'internal'
+ const upgrade = error.allowance.plan === 'internal' || error.allowance.plan === 'super_admin'
? null
: getHostedAiCapacityUpgrade(error.allowance.plan);
const payload = {
diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.ts
index d7c9b99b4..1e840d266 100644
--- a/packages/ai-gateway/src/index.ts
+++ b/packages/ai-gateway/src/index.ts
@@ -662,7 +662,9 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
served_tier: response.headers.get('x-screenpipe-served-tier'),
router_tier: routerTier,
workload: latency,
- gateway_mode: 'cloudflare',
+ gateway_mode: response.headers.has('x-screenpipe-background-fallback') ? 'direct' : 'cloudflare',
+ fallback_model: response.headers.get('x-screenpipe-background-fallback'),
+ fallback_reason: response.headers.get('x-screenpipe-background-fallback-reason'),
latency_ms: latencyMs,
status_code: response.status,
});
@@ -681,7 +683,7 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
if (response.ok && body.stream) {
const { response: trackedResponse, usage: usagePromise } = trackResponseUsage(response, 'openai');
response = trackedResponse;
- void usagePromise.then(u => logCost(env, {
+ ctx.waitUntil(usagePromise.then(u => logCost(env, {
device_id: authResult.deviceId,
user_id: authResult.userId,
tier: authResult.tier,
@@ -704,9 +706,9 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
stream: true,
latency_ms: latencyMs,
router_tier: routerTier,
- }));
+ })));
} else if (response.ok) {
- void settleActualOrReservedCost(
+ ctx.waitUntil(settleActualOrReservedCost(
env,
null,
reservedCostAttribution(
@@ -747,7 +749,7 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
router_tier: routerTier,
});
},
- );
+ ));
}
return attachLeaseRelease(response);
diff --git a/packages/ai-gateway/src/services/api-audit.ts b/packages/ai-gateway/src/services/api-audit.ts
index 8de439038..6b5b19b18 100644
--- a/packages/ai-gateway/src/services/api-audit.ts
+++ b/packages/ai-gateway/src/services/api-audit.ts
@@ -35,7 +35,9 @@ export interface ApiRouteAudit {
served_tier?: string | null;
router_tier?: string | null;
workload: 'interactive' | 'background';
- gateway_mode: 'cloudflare';
+ gateway_mode: 'cloudflare' | 'direct';
+ fallback_model?: string | null;
+ fallback_reason?: string | null;
latency_ms: number;
status_code: number;
}
diff --git a/packages/ai-gateway/src/services/background-limit-fallback.ts b/packages/ai-gateway/src/services/background-limit-fallback.ts
index b43929b08..7694e9890 100644
--- a/packages/ai-gateway/src/services/background-limit-fallback.ts
+++ b/packages/ai-gateway/src/services/background-limit-fallback.ts
@@ -5,21 +5,15 @@
import type { Env, RequestBody } from '../types';
import { isHostedChatAllowanceError } from './cloudflare-ai-gateway';
+/** Existing rescue for provider quota and safety responses. */
+export const BACKGROUND_FALLBACK_MODEL = 'gpt-5.4-nano';
+
/**
- * Rescue model for unattended background Pipes whose account allowance is gone.
- *
- * This lane deliberately calls the provider directly (the caller passes no
- * Cloudflare gateway context), so it is not blocked by the same spend limit
- * that just rejected the primary request.
- *
- * Previously this was `argus-trace-1` (self-hosted Qwen). Its 8k window and
- * 512-token output ceiling forced heavy prompt truncation, and in practice it
- * returned HTTP 200 with no structured tool calls — a silent failure that a
- * Pipe records as a successful run. gpt-5.4-nano is ~$0.20/$1.25 per Mtok with
- * a normal context window and reliable tool calling, which is cheap enough to
- * serve over-allowance work without the false-success failure mode.
+ * Account allowance rejection happens before inference. Eligible background
+ * requests can use Luna directly without changing model quality or paying the
+ * higher nano token rate. Provider failures retain their separate rescue path.
*/
-export const BACKGROUND_FALLBACK_MODEL = 'gpt-5.4-nano';
+export const ALLOWANCE_FALLBACK_MODEL = 'gpt-6-luna';
export class SafetyRefusalError extends Error {
readonly code = 'safety_refusal';
@@ -63,14 +57,10 @@ export function hasUnsupportedFallbackInput(body: RequestBody): boolean {
/**
* Swap in the rescue model and otherwise preserve the Pipe's request exactly.
- *
- * The old Argus lane rewrote roles, clipped tool descriptions, truncated the
- * oldest context, and clamped output to 512 tokens. All of that existed only to
- * fit an 8k self-hosted window. gpt-5.4-nano takes the request as Pi built it,
- * so the rescued run behaves like the primary one.
+ * Both rescue models support the original context, tool schema and output budget.
*/
-export function prepareBackgroundFallbackBody(body: RequestBody): RequestBody {
- return { ...body, model: BACKGROUND_FALLBACK_MODEL };
+export function prepareBackgroundFallbackBody(body: RequestBody, model = BACKGROUND_FALLBACK_MODEL): RequestBody {
+ return { ...body, model };
}
function errorText(error: unknown): string {
@@ -143,7 +133,10 @@ export function shouldUseBackgroundFallback(input: {
env: Pick<Env, 'OPENAI_API_KEY'>;
}): boolean {
return input.enabled &&
- input.body.model !== BACKGROUND_FALLBACK_MODEL &&
+ // An allowance rejected the gateway route, not the model. One direct
+ // attempt is valid even when the caller explicitly selected that model.
+ ((isHostedChatAllowanceError(input.error) || isAccountLocalAllowanceError(input.error))
+ || input.body.model !== BACKGROUND_FALLBACK_MODEL) &&
!hasUnsupportedFallbackInput(input.body) &&
isBackgroundFallbackConfigured(input.env) &&
(isHostedChatAllowanceError(input.error) ||
@@ -164,5 +157,9 @@ export function resolveBackgroundFallbackBody(input: {
env: Pick<Env, 'OPENAI_API_KEY'>;
}): RequestBody | null {
if (!shouldUseBackgroundFallback(input)) return null;
- return prepareBackgroundFallbackBody(input.body);
+ return prepareBackgroundFallbackBody(input.body,
+ isHostedChatAllowanceError(input.error) || isAccountLocalAllowanceError(input.error)
+ ? ALLOWANCE_FALLBACK_MODEL
+ : BACKGROUND_FALLBACK_MODEL,
+ );
}
diff --git a/packages/ai-gateway/src/test/allowance-rescue-route.unit.test.ts b/packages/ai-gateway/src/test/allowance-rescue-route.unit.test.ts
new file mode 100644
index 000000000..2828eb10a
--- /dev/null
+++ b/packages/ai-gateway/src/test/allowance-rescue-route.unit.test.ts
@@ -0,0 +1,81 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { describe, expect, it, spyOn } from 'bun:test';
+import { handleRequest } from '../index';
+import type { Env, RequestBody } from '../types';
+import { OpenAIProvider } from '../providers/openai';
+
+
+describe('allowance rescue through the chat route', () => {
+ for (const model of ['auto', 'gpt-6-luna']) {
+ for (const stream of [false, true]) {
+ it(`rescues ${model} (${stream ? 'SSE' : 'JSON'}) once and retains cost writes after the response`, async () => {
+ const calls: Array<{ url: string; body: any; headers: Headers }> = [];
+ const health: unknown[][] = [];
+ const costs: unknown[][] = [];
+ const lifetime: Promise<unknown>[] = [];
+ let releaseWrite!: () => void;
+ const writeGate = new Promise<void>((resolve) => { releaseWrite = resolve; });
+ const info = spyOn(console, 'info').mockImplementation(() => {});
+ const warn = spyOn(console, 'warn').mockImplementation(() => {});
+ const toolCall = { id: 'synthetic-call', type: 'function', function: { name: 'lookup', arguments: '{"query":"synthetic"}' } };
+ const usage = { prompt_tokens: 100, completion_tokens: 20, prompt_tokens_details: { cached_tokens: 40 } };
+ const reply = async (provider: OpenAIProvider, body: RequestBody) => {
+ const client = (provider as any).client;
+ calls.push({ url: client.baseURL, body, headers: new Headers(client._options.defaultHeaders) });
+ if (client.baseURL.startsWith('https://gateway.ai.cloudflare.com/')) throw Object.assign(new Error('Spend limit exceeded'), { status: 429 });
+ if (client.baseURL !== 'https://api.openai.com/v1') throw new Error('unexpected provider');
+ const json = { model: body.model, choices: [{ message: { role: 'assistant', content: null, tool_calls: [toolCall] }, finish_reason: 'tool_calls' }], usage };
+ const sse = [
+ { choices: [{ delta: { role: 'assistant', tool_calls: [{ index: 0, ...toolCall }] }, finish_reason: null }] },
+ { choices: [{ delta: {}, finish_reason: 'tool_calls' }] },
+ { choices: [], usage },
+ ].map((event) => `data: ${JSON.stringify(event)}\n\n`).join('') + 'data: [DONE]\n\n';
+ return stream ? new Response(sse, { headers: { 'content-type': 'text/event-stream' } }) : Response.json(json);
+ };
+ const jsonSpy = spyOn(OpenAIProvider.prototype, 'createCompletion').mockImplementation(function (body) { return reply(this, body); });
+ const streamSpy = spyOn(OpenAIProvider.prototype, 'createStreamingCompletion').mockImplementation(async function (body) { return (await reply(this, body)).body!; });
+ const env = {
+ AI_GATEWAY_SERVICE_TOKEN: 'synthetic-service-token', OPENAI_API_KEY: 'synthetic-openai-key',
+ CLOUDFLARE_AI_GATEWAY_ID: 'synthetic', CLOUDFLARE_AI_GATEWAY_TOKEN: 'synthetic-gateway-key',
+ CLOUDFLARE_AI_GATEWAY_BASE_URL: 'https://gateway.ai.cloudflare.com/v1/00000000000000000000000000000000/synthetic/compat/chat/completions',
+ RATE_LIMITER: { idFromName: (name: string) => name, get: () => ({ fetch: async () => Response.json({ allowed: true, standing: 'good' }) }) },
+ DB: { prepare: (sql: string) => ({ bind: (...values: unknown[]) => ({
+ first: async () => null,
+ run: async () => {
+ if (sql.includes('INSERT INTO model_health_window')) health.push(values);
+ if (sql.includes('INSERT INTO cost_daily')) { await writeGate; costs.push(values); }
+ return { success: true };
+ },
+ }) }) },
+ } as unknown as Env;
+ try {
+ const response = await handleRequest(new Request('https://gateway.test/v1/chat/completions', {
+ method: 'POST', headers: { authorization: 'Bearer synthetic-service-token', 'content-type': 'application/json', 'x-screenpipe-workload': 'pipe', 'x-screenpipe-latency': 'background' },
+ body: JSON.stringify({ model, stream, messages: [{ role: 'user', content: 'Synthetic lookup' }], tools: [{ type: 'function', function: { name: 'lookup', parameters: { type: 'object', properties: { query: { type: 'string' } } } } }], max_completion_tokens: 64 }),
+ }), env, { waitUntil: (task: Promise<unknown>) => lifetime.push(task), passThroughOnException() {} } as unknown as ExecutionContext);
+ expect(response.status).toBe(200);
+ expect(response.headers.get('x-screenpipe-background-fallback')).toBe('gpt-6-luna');
+ expect(await response.text()).toContain('synthetic-call');
+ expect(calls.map((call) => call.body.model)).toEqual(['gpt-6-luna', 'gpt-6-luna']);
+ expect(calls[0].headers.has('cf-aig-metadata')).toBe(true);
+ expect(calls[1].headers.has('cf-aig-metadata')).toBe(false);
+ expect(calls[1].body.tools[0].function.name).toBe('lookup');
+ expect(health.some((values) => values[1] === 'rate_limited')).toBe(false);
+ const route = info.mock.calls.map((args) => String(args[0])).find((line) => line.startsWith('screenpipe.ai-gateway-route '));
+ expect(JSON.parse(route!.slice('screenpipe.ai-gateway-route '.length))).toMatchObject({ gateway_mode: 'direct', fallback_model: 'gpt-6-luna', fallback_reason: 'account_allowance', status_code: 200 });
+ expect(costs).toHaveLength(0);
+ // Cost I/O is deliberately unfinished after the body is delivered.
+ // It must be attached to the request lifetime, not fire-and-forget.
+ expect(lifetime.length).toBeGreaterThan(0);
+ releaseWrite();
+ await Promise.all(lifetime);
+ expect(costs).toHaveLength(1);
+ expect(costs[0][3]).toBe('gpt-6-luna');
+ expect(costs[0].slice(7, 10)).toEqual([100, 20, 40]);
+ } finally { releaseWrite(); await Promise.allSettled(lifetime); info.mockRestore(); warn.mockRestore(); jsonSpy.mockRestore(); streamSpy.mockRestore(); }
+ });
+ }
+ }
+});
diff --git a/packages/ai-gateway/src/test/background-limit-fallback.unit.test.ts b/packages/ai-gateway/src/test/background-limit-fallback.unit.test.ts
index 4ae18689f..3731051fd 100644
--- a/packages/ai-gateway/src/test/background-limit-fallback.unit.test.ts
+++ b/packages/ai-gateway/src/test/background-limit-fallback.unit.test.ts
@@ -6,6 +6,7 @@ import { describe, expect, it, mock } from 'bun:test';
import { runChain, tryBackgroundFallback, trySafetyRefusalFallback } from '../handlers/chat';
import {
BACKGROUND_FALLBACK_MODEL,
+ ALLOWANCE_FALLBACK_MODEL,
SafetyRefusalError,
hasUnsupportedFallbackInput,
isAccountLocalAllowanceError,
@@ -119,7 +120,7 @@ describe('paid background Pipe rescue fallback', () => {
});
expect(fallbackBody).not.toBeNull();
- expect(fallbackBody?.model).toBe(BACKGROUND_FALLBACK_MODEL);
+ expect(fallbackBody?.model).toBe(ALLOWANCE_FALLBACK_MODEL);
// Pi's own `developer` role and full output budget survive the swap.
expect(fallbackBody?.messages[0].role).toBe('developer');
expect(fallbackBody?.max_completion_tokens).toBe(32_000);
@@ -141,8 +142,8 @@ describe('paid background Pipe rescue fallback', () => {
_flex: boolean,
gatewayContext: unknown,
) => {
- expect(model).toBe(BACKGROUND_FALLBACK_MODEL);
- expect(request.model).toBe(BACKGROUND_FALLBACK_MODEL);
+ expect(model).toBe(ALLOWANCE_FALLBACK_MODEL);
+ expect(request.model).toBe(ALLOWANCE_FALLBACK_MODEL);
expect(request.stream).toBe(stream);
expect(gatewayContext).toBeUndefined();
return new Response(stream ? 'data: [DONE]\n\n' : JSON.stringify({ choices: [] }));
@@ -155,8 +156,8 @@ describe('paid background Pipe rescue fallback', () => {
attempt as any,
);
expect(response?.status).toBe(200);
- expect(response?.headers.get('x-screenpipe-model')).toBe(BACKGROUND_FALLBACK_MODEL);
- expect(response?.headers.get('x-screenpipe-background-fallback')).toBe(BACKGROUND_FALLBACK_MODEL);
+ expect(response?.headers.get('x-screenpipe-model')).toBe(ALLOWANCE_FALLBACK_MODEL);
+ expect(response?.headers.get('x-screenpipe-background-fallback')).toBe(ALLOWANCE_FALLBACK_MODEL);
expect(attempt).toHaveBeenCalledTimes(1);
}
});
diff --git a/packages/ai-gateway/src/test/cloudflare-chat-routing.unit.test.ts b/packages/ai-gateway/src/test/cloudflare-chat-routing.unit.test.ts
index 255a1ae57..5a8c71d5c 100644
--- a/packages/ai-gateway/src/test/cloudflare-chat-routing.unit.test.ts
+++ b/packages/ai-gateway/src/test/cloudflare-chat-routing.unit.test.ts
@@ -24,6 +24,13 @@ const body: RequestBody = {
};
describe('Cloudflare hosted-chat routing', () => {
+ it.each(['internal', 'super_admin'] as const)('preserves the %s allowance error without an upgrade offer', async (plan) => {
+ const response = allowanceErrorResponse(body, new HostedChatAllowanceExceededError({
+ user_id: 'synthetic', plan, lane: 'auto', workload: 'background',
+ }));
+ expect(response.status).toBe(429);
+ expect(await response.json()).toMatchObject({ required_plan: null, upgrade_url: null });
+ });
it.each(['combined', 'unknown'] as const)('treats a %s spend-limit error as terminal without provider cascade', async (limitScope) => {
const context = await buildHostedChatGatewayContext(basicAuth, body.model, 'interactive');
const attempts = mock(async () => {
diff --git a/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts b/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts
index 0e31b5b9d..f8bddb61f 100644
--- a/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts
+++ b/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts
@@ -20,6 +20,68 @@ afterEach(async () => {
});
describe('local AI gateway harness', () => {
+ test('recovers allowance blocks on direct Luna with real SDK tool streaming and D1 accounting', async () => {
+ const usage = { prompt_tokens: 100, completion_tokens: 20, prompt_tokens_details: { cached_tokens: 40 } };
+ const call = { id: 'synthetic-lookup', type: 'function', function: { name: 'lookup', arguments: '{"query":"synthetic"}' } };
+ const harness = await startHarness({ outboundResponse: async (request, body) => {
+ if (request.url.startsWith('https://gateway.ai.cloudflare.com/')) {
+ return Response.json({ error: { message: 'Spend limit exceeded', code: 2041 } }, { status: 429 });
+ }
+ if (request.url === 'https://api.openai.com/v1/chat/completions') {
+ expect(body.model).toBe('gpt-6-luna');
+ expect(body.reasoning_effort).toBe('none');
+ expect(body.tools[0].function.name).toBe('lookup');
+ expect(request.headers.has('cf-aig-metadata')).toBe(false);
+ if (!body.stream) return Response.json({ model: body.model, choices: [{ message: { role: 'assistant', content: null, tool_calls: [call] }, finish_reason: 'tool_calls' }], usage });
+ return new Response([
+ { choices: [{ delta: { role: 'assistant', tool_calls: [{ index: 0, ...call }] }, finish_reason: null }] },
+ { choices: [{ delta: {}, finish_reason: 'tool_calls' }] },
+ { choices: [], usage },
+ ].map((event) => `data: ${JSON.stringify(event)}\n\n`).join('') + 'data: [DONE]\n\n', { headers: { 'content-type': 'text/event-stream' } });
+ }
+ } });
+ for (const model of ['auto', 'gpt-6-luna']) {
+ for (const stream of [false, true]) {
+ const response = await harness.fetch('/chat/completions', {
+ method: 'POST', headers: { 'content-type': 'application/json', 'x-screenpipe-latency': 'background' },
+ body: JSON.stringify({ model, stream, messages: [{ role: 'user', content: 'Synthetic lookup' }], tools: [{ type: 'function', function: { name: 'lookup', parameters: { type: 'object', properties: { query: { type: 'string' } } } } }] }),
+ });
+ expect(response.status).toBe(200);
+ expect(response.headers.get('x-screenpipe-model')).toBe('gpt-6-luna');
+ expect(response.headers.get('x-screenpipe-background-fallback-reason')).toBe('account_allowance');
+ expect(await response.text()).toContain('synthetic-lookup');
+ }
+ }
+ expect(harness.outboundRequests).toHaveLength(8);
+ let telemetry = await harness.readInferenceTelemetry();
+ for (let attempt = 0; attempt < 20 && telemetry.costs[0]?.requests !== 4; attempt++) {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ telemetry = await harness.readInferenceTelemetry();
+ }
+ expect(telemetry.costs).toEqual([{ model: 'gpt-6-luna', requests: 4, input_tokens: 400, output_tokens: 80, cost: expect.closeTo(0.0000656, 10) }]);
+ expect(telemetry.health.some((row) => row.outcome === 'rate_limited')).toBe(false);
+ harness.assertNoUnexpectedOutboundRequests();
+ });
+
+ test('preserves allowance errors for interactive, image, and failed direct-rescue requests', async () => {
+ const harness = await startHarness({ outboundResponse: async (request) => {
+ if (request.url.startsWith('https://gateway.ai.cloudflare.com/')) return Response.json({ error: { message: 'Spend limit exceeded' } }, { status: 429 });
+ if (request.url === 'https://api.openai.com/v1/chat/completions') return Response.json({ error: { message: 'Rate limit reached: requests per minute' } }, { status: 429, headers: { 'x-should-retry': 'false' } });
+ } });
+ for (const variant of ['interactive', 'image', 'rescue-fails']) {
+ const response = await harness.fetch('/chat/completions', {
+ method: 'POST', headers: { 'content-type': 'application/json', ...(variant === 'interactive' ? {} : { 'x-screenpipe-latency': 'background' }) },
+ body: JSON.stringify({ model: 'auto', messages: [{ role: 'user', content: variant === 'image' ? [{ type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } }] : 'Synthetic request' }] }),
+ });
+ expect(response.status).toBe(429);
+ expect(await response.text()).toContain('hosted_ai_allowance_exceeded');
+ }
+ expect(harness.outboundRequests.filter((request) => request.url.startsWith('https://api.openai.com/'))).toHaveLength(1);
+ const telemetry = await harness.readInferenceTelemetry();
+ expect(telemetry.costs).toEqual([]);
+ expect(telemetry.health).toEqual([{ model: 'gpt-6-luna', outcome: 'rate_limited', requests: 1 }]);
+ harness.assertNoUnexpectedOutboundRequests();
+ });
test('preserves voice responses across the real Durable Object fetch boundary', async () => {
const harness = await startHarness({
outboundResponse: async (request) => {
diff --git a/packages/ai-gateway/src/test/local-gateway-harness.ts b/packages/ai-gateway/src/test/local-gateway-harness.ts
index 292a43614..3887fb0aa 100644
--- a/packages/ai-gateway/src/test/local-gateway-harness.ts
+++ b/packages/ai-gateway/src/test/local-gateway-harness.ts
@@ -402,6 +402,15 @@ export class LocalGatewayHarness {
};
}
+ async readInferenceTelemetry() {
+ if (!this.database) throw new Error('local AI gateway database is not ready');
+ const [costs, health] = await Promise.all([
+ this.database.prepare('SELECT model, SUM(requests) AS requests, SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens, SUM(estimated_cost_usd) AS cost FROM cost_daily GROUP BY model').all(),
+ this.database.prepare('SELECT model, outcome, SUM(requests) AS requests FROM model_health_window GROUP BY model, outcome').all(),
+ ]);
+ return { costs: costs.results, health: health.results };
+ }
+
assertNoUnexpectedOutboundRequests(): void {
const unexpected = this.unexpectedOutboundRequests;
if (unexpected.length > 0) {