YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(ai-gateway): keep background pipes from blocking chat
Louis Beaumont3ad524fcd8af
diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.tsindex a31baea87..ccc34227d 100644--- a/packages/ai-gateway/src/index.ts+++ b/packages/ai-gateway/src/index.ts@@ -122,7 +122,14 @@ async function handleMeteredTinfoilRequest(subPath: '/v1/chat/completions' | '/v1/responses',): Promise<Response> {const model = 'gemma4-31b';- const reservation = await reserveDailyCostCap(env, auth.deviceId, auth.tier, model);+ const reservation = await reserveDailyCostCap(+ env,+ auth.deviceId,+ auth.tier,+ model,+ new Date(),+ isBackgroundRequest(request) ? 'background' : 'interactive',+ );if (!reservation.allowed) return reservation.response;let response: Response;try {@@ -158,7 +165,14 @@ async function handleMeteredVoiceAiRequest(endpoint: '/v1/voice/query' | '/v1/voice/chat',): Promise<Response> {const model = request.headers.get('ai-model') || 'gpt-5.4';- const reservation = await reserveDailyCostCap(env, auth.deviceId, auth.tier, model);+ const reservation = await reserveDailyCostCap(+ env,+ auth.deviceId,+ auth.tier,+ model,+ new Date(),+ isBackgroundRequest(request) ? 'background' : 'interactive',+ );if (!reservation.allowed) return reservation.response;let response: Response;try {@@ -410,13 +424,16 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCofreeChatLease = reservation.lease;}- // Serialize the account's priced read/inference/write interval. Acquiring- // this only after every non-cost gate avoids holding it for rejected work.+ // Serialize priced work within its foreground/background lane. A scheduled+ // pipe must not block a user who is actively waiting in chat.+ const latency = resolveLatencyClass(request, body, env);const costReservation = await reserveDailyCostCap(env,authResult.deviceId,authResult.tier,body.model,+ new Date(),+ isBackgroundRequest(request) ? 'background' : 'interactive',);if (!costReservation.allowed) {if (freeChatLease) await releaseFreeChatLease(env, freeChatLease);@@ -425,7 +442,6 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCoconst dailyCostLease = costReservation.lease;// Route latency-tolerant (background) traffic to the cheaper flex tier.- const latency = resolveLatencyClass(request, body, env);let leaseReleased = false;const releaseLease = async () => {if (!freeChatLease || leaseReleased) return;@@ -576,6 +592,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCoauthResult.deviceId,authResult.tier,'gemini-2.5-flash',+ new Date(),+ isBackgroundRequest(request) ? 'background' : 'interactive',);if (!costReservation.allowed) return costReservation.response;const webSearchResponse = await handleWebSearch(request, env);@@ -755,6 +773,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCoauthResult.deviceId,authResult.tier,parsedModel,+ new Date(),+ isBackgroundRequest(request) ? 'background' : 'interactive',);if (!costReservation.allowed) return costReservation.response;@@ -891,6 +911,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCoauthResult.deviceId,authResult.tier,ocModel,+ new Date(),+ isBackgroundRequest(request) ? 'background' : 'interactive',);if (!costReservation.allowed) return costReservation.response;diff --git a/packages/ai-gateway/src/services/cost-cap.ts b/packages/ai-gateway/src/services/cost-cap.tsindex f5f3b9b51..5e5f2d78d 100644--- a/packages/ai-gateway/src/services/cost-cap.ts+++ b/packages/ai-gateway/src/services/cost-cap.ts@@ -23,6 +23,8 @@ export type DailyCostLease = {expiresAt: string;};+export type DailyCostLane = 'interactive' | 'background';+export type DailyCostReservation =| { allowed: true; lease: DailyCostLease | null }| { allowed: false; response: Response };@@ -176,13 +178,18 @@ export async function reserveDailyCostCap(tier: string,model: string,now: Date = new Date(),+ lane: DailyCostLane = 'interactive',): Promise<DailyCostReservation> {if (isZeroCostModel(model)) return { allowed: true, lease: null };let lease: DailyCostLease | null = null;try {const leaseEpoch = configuredCostCapEpoch(env) ?? 'legacy';- const key = `daily-cost:lease:v1:${await sha256Hex(`${leaseEpoch}:${deviceId}`)}`;+ // Foreground chat must not be rejected merely because a scheduled pipe is+ // running. Keep one priced request per lane: background remains serialized,+ // while an interactive request can proceed alongside it. The account-wide+ // cash accumulator and cap remain shared across both lanes.+ const key = `daily-cost:lease:v2:${await sha256Hex(`${leaseEpoch}:${deviceId}:${lane}`)}`;const nowIso = now.toISOString();const expiresAt = new Date(now.getTime() + DAILY_COST_LEASE_SECONDS * 1000).toISOString();const claim = async () => env.DB.prepare(`@@ -205,7 +212,9 @@ export async function reserveDailyCostCap(allowed: false,response: addCorsHeaders(createErrorResponse(429, JSON.stringify({error: 'priced_request_in_flight',- message: 'Another hosted AI request is still running for this account. Wait for it to finish before retrying.',+ message: lane === 'background'+ ? 'Another hosted AI background request is still running for this account. Wait for it to finish before retrying.'+ : 'Another hosted AI chat request is still running for this account. Wait for it to finish before retrying.',}))),};}diff --git a/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts b/packages/ai-gateway/src/services/free-chat-limit.integration.test.tsindex da2a7d471..e2a9f699e 100644--- a/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts+++ b/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts@@ -205,6 +205,25 @@ describe('usage reservations against workerd D1', () => {)).allowed).toBe(true);});+ it('grants one priced-request lease in each foreground/background lane', async () => {+ const now = new Date('2026-07-14T12:00:00.000Z');+ const background = await reserveDailyCostCap(+ env, 'user-d1-cost-lanes', 'subscribed', 'claude-sonnet-5', now, 'background',+ );+ const interactive = await reserveDailyCostCap(+ env, 'user-d1-cost-lanes', 'subscribed', 'claude-sonnet-5', now, 'interactive',+ );+ const overlap = await reserveDailyCostCap(+ env, 'user-d1-cost-lanes', 'subscribed', 'claude-sonnet-5', now, 'background',+ );++ expect(background.allowed).toBe(true);+ expect(interactive.allowed).toBe(true);+ expect(overlap.allowed).toBe(false);+ if (background.allowed && background.lease) await releaseDailyCostLease(env, background.lease);+ if (interactive.allowed && interactive.lease) await releaseDailyCostLease(env, interactive.lease);+ });+it('reclaims an expired priced-request lease without accepting its stale release', async () => {const start = new Date('2026-07-14T12:00:00.000Z');const first = await reserveDailyCostCap(env, 'user-d1-cost-expired', 'subscribed', 'gpt-5.6-sol', start);diff --git a/packages/ai-gateway/src/test/cost-cap.unit.test.ts b/packages/ai-gateway/src/test/cost-cap.unit.test.tsindex 444bb09e5..2e18fb6fb 100644--- a/packages/ai-gateway/src/test/cost-cap.unit.test.ts+++ b/packages/ai-gateway/src/test/cost-cap.unit.test.ts@@ -202,6 +202,28 @@ describe('reserveDailyCostCap', () => {}});+ it('lets foreground chat proceed while a background pipe is running', async () => {+ const db = new LeaseD1();+ const env = leaseEnv(db);+ const now = new Date('2026-07-30T12:00:00.000Z');+ const background = await reserveDailyCostCap(+ env, 'user_lanes', 'subscribed', 'claude-sonnet-5', now, 'background',+ );+ const interactive = await reserveDailyCostCap(+ env, 'user_lanes', 'subscribed', 'claude-sonnet-5', now, 'interactive',+ );+ const secondBackground = await reserveDailyCostCap(+ env, 'user_lanes', 'subscribed', 'claude-sonnet-5', now, 'background',+ );++ expect(background.allowed).toBe(true);+ expect(interactive.allowed).toBe(true);+ expect(secondBackground.allowed).toBe(false);+ if (!secondBackground.allowed) {+ expect(await secondBackground.response.text()).toContain('background request');+ }+ });+it('allows the next request only after the exact lease generation releases', async () => {const db = new LeaseD1();const env = leaseEnv(db);