Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai-gateway): keep background pipes from blocking chat

Louis Beaumont3ad524fcd8af
diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.ts
index a31baea87..ccc34227d 100644
--- a/packages/ai-gateway/src/index.ts
+++ b/packages/ai-gateway/src/index.ts
@@ -122,7 +122,14 @@ async function handleMeteredTinfoilRequest(
subPath: '/v1/chat/completions' | '/v1/responses',
): Promise<Response> {
const model = 'gemma4-31b';
- const reservation = await reserveDailyCostCap(env, auth.deviceId, auth.tier, model);
+ const reservation = await reserveDailyCostCap(
+ env,
+ auth.deviceId,
+ auth.tier,
+ model,
+ new Date(),
+ isBackgroundRequest(request) ? 'background' : 'interactive',
+ );
if (!reservation.allowed) return reservation.response;
let response: Response;
try {
@@ -158,7 +165,14 @@ async function handleMeteredVoiceAiRequest(
endpoint: '/v1/voice/query' | '/v1/voice/chat',
): Promise<Response> {
const model = request.headers.get('ai-model') || 'gpt-5.4';
- const reservation = await reserveDailyCostCap(env, auth.deviceId, auth.tier, model);
+ const reservation = await reserveDailyCostCap(
+ env,
+ auth.deviceId,
+ auth.tier,
+ model,
+ new Date(),
+ isBackgroundRequest(request) ? 'background' : 'interactive',
+ );
if (!reservation.allowed) return reservation.response;
let response: Response;
try {
@@ -410,13 +424,16 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
freeChatLease = reservation.lease;
}
- // Serialize the account's priced read/inference/write interval. Acquiring
- // this only after every non-cost gate avoids holding it for rejected work.
+ // Serialize priced work within its foreground/background lane. A scheduled
+ // pipe must not block a user who is actively waiting in chat.
+ const latency = resolveLatencyClass(request, body, env);
const costReservation = await reserveDailyCostCap(
env,
authResult.deviceId,
authResult.tier,
body.model,
+ new Date(),
+ isBackgroundRequest(request) ? 'background' : 'interactive',
);
if (!costReservation.allowed) {
if (freeChatLease) await releaseFreeChatLease(env, freeChatLease);
@@ -425,7 +442,6 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
const dailyCostLease = costReservation.lease;
// Route latency-tolerant (background) traffic to the cheaper flex tier.
- const latency = resolveLatencyClass(request, body, env);
let leaseReleased = false;
const releaseLease = async () => {
if (!freeChatLease || leaseReleased) return;
@@ -576,6 +592,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
authResult.deviceId,
authResult.tier,
'gemini-2.5-flash',
+ new Date(),
+ isBackgroundRequest(request) ? 'background' : 'interactive',
);
if (!costReservation.allowed) return costReservation.response;
const webSearchResponse = await handleWebSearch(request, env);
@@ -755,6 +773,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
authResult.deviceId,
authResult.tier,
parsedModel,
+ new Date(),
+ isBackgroundRequest(request) ? 'background' : 'interactive',
);
if (!costReservation.allowed) return costReservation.response;
@@ -891,6 +911,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
authResult.deviceId,
authResult.tier,
ocModel,
+ new Date(),
+ isBackgroundRequest(request) ? 'background' : 'interactive',
);
if (!costReservation.allowed) return costReservation.response;
diff --git a/packages/ai-gateway/src/services/cost-cap.ts b/packages/ai-gateway/src/services/cost-cap.ts
index f5f3b9b51..5e5f2d78d 100644
--- a/packages/ai-gateway/src/services/cost-cap.ts
+++ b/packages/ai-gateway/src/services/cost-cap.ts
@@ -23,6 +23,8 @@ export type DailyCostLease = {
expiresAt: string;
};
+export type DailyCostLane = 'interactive' | 'background';
+
export type DailyCostReservation =
| { allowed: true; lease: DailyCostLease | null }
| { allowed: false; response: Response };
@@ -176,13 +178,18 @@ export async function reserveDailyCostCap(
tier: string,
model: string,
now: Date = new Date(),
+ lane: DailyCostLane = 'interactive',
): Promise<DailyCostReservation> {
if (isZeroCostModel(model)) return { allowed: true, lease: null };
let lease: DailyCostLease | null = null;
try {
const leaseEpoch = configuredCostCapEpoch(env) ?? 'legacy';
- const key = `daily-cost:lease:v1:${await sha256Hex(`${leaseEpoch}:${deviceId}`)}`;
+ // Foreground chat must not be rejected merely because a scheduled pipe is
+ // running. Keep one priced request per lane: background remains serialized,
+ // while an interactive request can proceed alongside it. The account-wide
+ // cash accumulator and cap remain shared across both lanes.
+ const key = `daily-cost:lease:v2:${await sha256Hex(`${leaseEpoch}:${deviceId}:${lane}`)}`;
const nowIso = now.toISOString();
const expiresAt = new Date(now.getTime() + DAILY_COST_LEASE_SECONDS * 1000).toISOString();
const claim = async () => env.DB.prepare(`
@@ -205,7 +212,9 @@ export async function reserveDailyCostCap(
allowed: false,
response: addCorsHeaders(createErrorResponse(429, JSON.stringify({
error: 'priced_request_in_flight',
- message: 'Another hosted AI request is still running for this account. Wait for it to finish before retrying.',
+ message: lane === 'background'
+ ? 'Another hosted AI background request is still running for this account. Wait for it to finish before retrying.'
+ : 'Another hosted AI chat request is still running for this account. Wait for it to finish before retrying.',
}))),
};
}
diff --git a/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts b/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts
index da2a7d471..e2a9f699e 100644
--- a/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts
+++ b/packages/ai-gateway/src/services/free-chat-limit.integration.test.ts
@@ -205,6 +205,25 @@ describe('usage reservations against workerd D1', () => {
)).allowed).toBe(true);
});
+ it('grants one priced-request lease in each foreground/background lane', async () => {
+ const now = new Date('2026-07-14T12:00:00.000Z');
+ const background = await reserveDailyCostCap(
+ env, 'user-d1-cost-lanes', 'subscribed', 'claude-sonnet-5', now, 'background',
+ );
+ const interactive = await reserveDailyCostCap(
+ env, 'user-d1-cost-lanes', 'subscribed', 'claude-sonnet-5', now, 'interactive',
+ );
+ const overlap = await reserveDailyCostCap(
+ env, 'user-d1-cost-lanes', 'subscribed', 'claude-sonnet-5', now, 'background',
+ );
+
+ expect(background.allowed).toBe(true);
+ expect(interactive.allowed).toBe(true);
+ expect(overlap.allowed).toBe(false);
+ if (background.allowed && background.lease) await releaseDailyCostLease(env, background.lease);
+ if (interactive.allowed && interactive.lease) await releaseDailyCostLease(env, interactive.lease);
+ });
+
it('reclaims an expired priced-request lease without accepting its stale release', async () => {
const start = new Date('2026-07-14T12:00:00.000Z');
const first = await reserveDailyCostCap(env, 'user-d1-cost-expired', 'subscribed', 'gpt-5.6-sol', start);
diff --git a/packages/ai-gateway/src/test/cost-cap.unit.test.ts b/packages/ai-gateway/src/test/cost-cap.unit.test.ts
index 444bb09e5..2e18fb6fb 100644
--- a/packages/ai-gateway/src/test/cost-cap.unit.test.ts
+++ b/packages/ai-gateway/src/test/cost-cap.unit.test.ts
@@ -202,6 +202,28 @@ describe('reserveDailyCostCap', () => {
}
});
+ it('lets foreground chat proceed while a background pipe is running', async () => {
+ const db = new LeaseD1();
+ const env = leaseEnv(db);
+ const now = new Date('2026-07-30T12:00:00.000Z');
+ const background = await reserveDailyCostCap(
+ env, 'user_lanes', 'subscribed', 'claude-sonnet-5', now, 'background',
+ );
+ const interactive = await reserveDailyCostCap(
+ env, 'user_lanes', 'subscribed', 'claude-sonnet-5', now, 'interactive',
+ );
+ const secondBackground = await reserveDailyCostCap(
+ env, 'user_lanes', 'subscribed', 'claude-sonnet-5', now, 'background',
+ );
+
+ expect(background.allowed).toBe(true);
+ expect(interactive.allowed).toBe(true);
+ expect(secondBackground.allowed).toBe(false);
+ if (!secondBackground.allowed) {
+ expect(await secondBackground.response.text()).toContain('background request');
+ }
+ });
+
it('allows the next request only after the exact lease generation releases', async () => {
const db = new LeaseD1();
const env = leaseEnv(db);