Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai-gateway): remove legacy hosted chat mode (#6890)

Ezra Ellette18661dfab746
diff --git a/packages/ai-gateway/.dev.vars.example b/packages/ai-gateway/.dev.vars.example
index 264e11faf..d995f618d 100644
--- a/packages/ai-gateway/.dev.vars.example
+++ b/packages/ai-gateway/.dev.vars.example
@@ -7,9 +7,8 @@ SENTRY_DSN=https://[email protected]/xxx
# Existing API keys
OPENAI_API_KEY=sk-...
ANTHROPIC_API_KEY=sk-ant-...
-# Hosted chat stays direct unless explicitly switched. The Gateway must exist
-# in the same Cloudflare account and have default OpenAI/Anthropic BYOK keys.
-HOSTED_CHAT_GATEWAY_MODE=legacy
+# Hosted chat always uses the Gateway. It must exist in the same Cloudflare
+# account and have default OpenAI/Anthropic BYOK keys.
CLOUDFLARE_AI_GATEWAY_ID=screenpipe-hosted-chat-staging
# Local development only. Use the token created in Gateway Settings when
# enabling Authenticated Gateway; Wrangler's OAuth token is not a Gateway run
diff --git a/packages/ai-gateway/src/handlers/models.ts b/packages/ai-gateway/src/handlers/models.ts
index d0d51f1c2..de040fc59 100644
--- a/packages/ai-gateway/src/handlers/models.ts
+++ b/packages/ai-gateway/src/handlers/models.ts
@@ -4,11 +4,10 @@
import { Env, UserTier, type AccountPlan } from '../types';
import { createSuccessResponse, createErrorResponse, addCorsHeaders } from '../utils/cors';
-import { getTierConfig, getModelWeight, isModelGatingEnabled } from '../services/usage-tracker';
+import { getTierConfig, isModelGatingEnabled } from '../services/usage-tracker';
import { getHostedAiAllowedModels, getHostedAiPlan } from '../services/hosted-ai-policy';
import { getModelHealth, ModelHealthStatus } from '../services/model-health';
import { isGooglePolicyBlockedModel } from '../utils/model-policy';
-import { isHostedChatGatewayEnabled } from '../services/cloudflare-ai-gateway';
/** Enriched model metadata — OpenAI-compatible (extra fields ignored by standard clients) */
interface ModelEntry {
@@ -50,7 +49,7 @@ interface ModelEntry {
* provider spend rules replace this proactive legacy query meter.
* UI uses `floor(remaining / query_weight)` to warn when the user is
* about to run out for a weighted model. Populated server-side from
- * `getModelWeight()` so client doesn't have to mirror the table.
+ * Cloudflare AI Gateway owns hosted-chat allowance, so this is always zero.
*/
query_weight?: number;
}
@@ -355,11 +354,10 @@ export async function handleModelListing(
// Avoid advertising models that would immediately fail because their
// provider secret is not configured in the Worker environment yet.
- const cloudflareGateway = isHostedChatGatewayEnabled(env);
models = models.filter(model =>
!model.requires_env ||
hasConfiguredSecret(env[model.requires_env]) ||
- (cloudflareGateway && model.requires_env === 'OPENAI_API_KEY')
+ model.requires_env === 'OPENAI_API_KEY'
);
models = models.filter(model => !isGooglePolicyBlockedModel(model.id));
@@ -385,7 +383,7 @@ export async function handleModelListing(
// Attach per-message query weight so UIs can warn the user before
// they run out for a weighted model. 0 means "doesn't count."
- model.query_weight = cloudflareGateway ? 0 : getModelWeight(model.id);
+ model.query_weight = 0;
}
const responseModels = models.map(({ requires_env, ...model }) => {
diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.ts
index bd7379225..869e6c644 100644
--- a/packages/ai-gateway/src/index.ts
+++ b/packages/ai-gateway/src/index.ts
@@ -24,7 +24,6 @@ import { handleVertexProxy, handleVertexModels } from './handlers/vertex-proxy';
import { handleWebSearch } from './handlers/web-search';
import { handleTinfoilAttestation, handleTinfoilProxy, parseTinfoilUsageMetrics } from './handlers/tinfoil-proxy';
import {
- getCostAccumulatorOrThrow,
getDailyUserCost,
getNonStreamSettlementCost,
getStreamSettlementCost,
@@ -32,15 +31,9 @@ import {
inferProvider,
isFrontierModel,
logCost,
- monthlyCostKey,
- trialCostKey,
resolveServedModel,
- utcMonth,
type CostReservationShape,
} from './services/cost-tracker';
-import {
- resolveHostedAiTextCostLimits,
-} from './services/hosted-ai-cost-controls';
import {
DEEPGRAM_FILE_COST_PER_HOUR,
readTranscribedSeconds,
@@ -57,7 +50,6 @@ import {
import {
reserveDailyCostCap,
withDailyCostSettlement,
- getDailyUserCostForCap,
type DailyCostHold,
} from './services/cost-cap';
import {
@@ -79,7 +71,6 @@ import {
import {
getHostedAiAllowedModels,
getHostedAiCapacityUpgrade,
- getHostedAiIncludedCredits,
getHostedAiPlan,
hasPaidHostedAiPlan,
isHostedAiUpgradeEligible,
@@ -95,14 +86,9 @@ import {
import { resolveModelAlias } from './providers';
import {
buildHostedChatGatewayContext,
- isHostedChatGatewayEnabled,
- shouldUseHostedChatGateway,
type HostedChatGatewayContext,
} from './services/cloudflare-ai-gateway';
import { getCloudflareHostedChatUsage } from './services/cloudflare-ai-gateway-usage';
-import {
- resolveBackgroundFallbackBody,
-} from './services/background-limit-fallback';
import { buildBackgroundPipeAllowanceAdvisory } from './services/background-pipe-advisory';
import { logApiAuthAudit, logApiRouteAudit } from './services/api-audit';
// import { handleTTSWebSocketUpgrade } from './handlers/voice-ws';
@@ -366,135 +352,64 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
const usageAccountPlan = authResult.tier === 'anonymous' && authResult.accountPlan === 'unknown'
? 'free'
: authResult.accountPlan;
- const cloudflareManaged = isHostedChatGatewayEnabled(env);
const status = await getUsageStatus(
env,
authResult.deviceId,
usageTier,
- cloudflareManaged ? undefined : authResult.userId,
+ undefined,
usageAccountPlan,
- { readLegacyDailyCounter: !cloudflareManaged },
+ { readLegacyDailyCounter: false },
);
- if (cloudflareManaged) {
- let cloudflareContext: HostedChatGatewayContext | null = null;
- let cloudflareUsage: Awaited<ReturnType<typeof getCloudflareHostedChatUsage>> = null;
- try {
- cloudflareContext = await buildHostedChatGatewayContext(
- authResult,
- 'auto',
- 'interactive',
- );
- cloudflareUsage = await getCloudflareHostedChatUsage(env, cloudflareContext);
- } catch (error) {
- // Hosted inference remains available when the read-only analytics
- // token or Cloudflare analytics is temporarily unavailable. Never
- // replace missing provider data with a fabricated zero balance.
- console.error('Cloudflare hosted AI usage unavailable', error);
- }
- const allowanceExhausted = cloudflareUsage?.allowances
- .some((allowance) => allowance.remaining_percent <= 0) ?? null;
- const totalAllowanceExhausted = cloudflareUsage?.allowances
- .some((allowance) =>
- allowance.lane === 'combined' && allowance.remaining_percent <= 0) ?? null;
- const capacityUpgrade = getHostedAiCapacityUpgrade(usageAccountPlan);
- const upgradeEligible = capacityUpgrade !== null;
- const enriched = {
- ...status,
- // Cloudflare owns the spend allowance in this mode. The
- // legacy query counters remain in the compatibility envelope, but
- // cannot be presented as a live provider-cost meter.
- upsell_banner: allowanceExhausted === true && upgradeEligible,
- cost_limit_reached: totalAllowanceExhausted,
- upgrade_eligible: upgradeEligible,
- hosted_ai: {
- // Use the exact plan sent to Cloudflare. Max and Ultra have
- // distinct allowance rules even though they share model access.
- plan: cloudflareContext?.plan ?? 'unknown',
- trial: authResult.hostedAiTrial === true,
- allowance_managed_by: 'cloudflare',
- included_credits: null,
- used_credits: null,
- remaining_credits: null,
- usage_as_of: cloudflareUsage?.usage_as_of ?? null,
- allowances: cloudflareUsage?.allowances ?? null,
- model_access: [...getHostedAiAllowedModels(usageAccountPlan)],
- frontier_models: getHostedAiAllowedModels(usageAccountPlan)
- .filter((model) => isFrontierModel(model)),
- required_plan: capacityUpgrade?.requiredPlan ?? null,
- upgrade_url: capacityUpgrade?.upgradeUrl ?? null,
- can_buy_credits: false,
- byok_supported: true,
- },
- background_pipe_advisory: buildBackgroundPipeAllowanceAdvisory({
- env,
- allowances: cloudflareUsage?.allowances ?? null,
- }),
- };
- return addCorsHeaders(createSuccessResponse(enriched));
- }
- // Enrich with cost-based limit flag (NOT the raw $ numbers — those
- // are our internal margin and shouldn't leak to any client/user).
- // Stored query credits do not raise the cash ceiling. Credit-funded
- // provider spend needs consumptive accounting before it can safely do so.
- const dailyCost = await getDailyUserCostForCap(env, authResult.deviceId);
- let maxCost: number;
- let monthlyCap: number;
+ let cloudflareContext: HostedChatGatewayContext | null = null;
+ let cloudflareUsage: Awaited<ReturnType<typeof getCloudflareHostedChatUsage>> = null;
try {
- const limits = resolveHostedAiTextCostLimits(
- usageAccountPlan,
- env,
- authResult.hostedAiTrial === true,
+ cloudflareContext = await buildHostedChatGatewayContext(
+ authResult,
+ 'auto',
+ 'interactive',
);
- maxCost = limits.daily;
- monthlyCap = limits.monthly;
+ cloudflareUsage = await getCloudflareHostedChatUsage(env, cloudflareContext);
} catch (error) {
- console.error('usage cost control configuration unavailable', error);
- return addCorsHeaders(createErrorResponse(503, JSON.stringify({
- error: 'cost_control_unavailable',
- message: 'AI usage controls are temporarily unavailable. Try again shortly.',
- })));
- }
- let monthlyCost: number | null = null;
- try {
- monthlyCost = await getCostAccumulatorOrThrow(
- env,
- authResult.hostedAiTrial === true
- ? trialCostKey(authResult.deviceId)
- : monthlyCostKey(authResult.deviceId),
- authResult.hostedAiTrial === true ? 'trial' : utcMonth(),
- );
- } catch {
- // Admission still fails closed. The status route stays available but
- // marks usage unknown instead of pretending the customer spent zero.
+ // Hosted inference remains available when the read-only analytics
+ // token or Cloudflare analytics is temporarily unavailable. Never
+ // replace missing provider data with a fabricated zero balance.
+ console.error('Cloudflare hosted AI usage unavailable', error);
}
- const includedCredits = getHostedAiIncludedCredits(usageAccountPlan);
- const usedCredits = monthlyCost === null ? null : Math.ceil(monthlyCost * 100);
+ const allowanceExhausted = cloudflareUsage?.allowances
+ .some((allowance) => allowance.remaining_percent <= 0) ?? null;
+ const totalAllowanceExhausted = cloudflareUsage?.allowances
+ .some((allowance) =>
+ allowance.lane === 'combined' && allowance.remaining_percent <= 0) ?? null;
const capacityUpgrade = getHostedAiCapacityUpgrade(usageAccountPlan);
+ const upgradeEligible = capacityUpgrade !== null;
const enriched = {
...status,
- cost_limit_reached: dailyCost >= maxCost || (monthlyCost !== null && monthlyCost >= monthlyCap),
- // This field controls proactive prompts. Capacity recovery is the
- // separate required_plan + upgrade_url contract below.
- upgrade_eligible: isHostedAiUpgradeEligible(authResult),
- upsell_banner: status.upsell_banner === true && isHostedAiUpgradeEligible(authResult),
+ upsell_banner: allowanceExhausted === true && upgradeEligible,
+ cost_limit_reached: totalAllowanceExhausted,
+ upgrade_eligible: upgradeEligible,
hosted_ai: {
- plan: getHostedAiPlan(usageAccountPlan) ?? 'unknown',
+ // Use the exact plan sent to Cloudflare. Max and Ultra have
+ // distinct allowance rules even though they share model access.
+ plan: cloudflareContext?.plan ?? getHostedAiPlan(usageAccountPlan) ?? 'unknown',
trial: authResult.hostedAiTrial === true,
- included_credits: includedCredits,
- used_credits: usedCredits,
- remaining_credits: usedCredits === null
- ? null
- : Math.max(0, includedCredits - usedCredits),
+ allowance_managed_by: 'cloudflare',
+ included_credits: null,
+ used_credits: null,
+ remaining_credits: null,
+ usage_as_of: cloudflareUsage?.usage_as_of ?? null,
+ allowances: cloudflareUsage?.allowances ?? null,
model_access: [...getHostedAiAllowedModels(usageAccountPlan)],
frontier_models: getHostedAiAllowedModels(usageAccountPlan)
.filter((model) => isFrontierModel(model)),
required_plan: capacityUpgrade?.requiredPlan ?? null,
upgrade_url: capacityUpgrade?.upgradeUrl ?? null,
- // Legacy query credits do not raise the provider-cost ceiling yet.
can_buy_credits: false,
byok_supported: true,
},
- background_pipe_advisory: null,
+ background_pipe_advisory: buildBackgroundPipeAllowanceAdvisory({
+ env,
+ allowances: cloudflareUsage?.allowances ?? null,
+ }),
};
return addCorsHeaders(createSuccessResponse(enriched));
}
@@ -626,8 +541,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
// already rewrote it to free 'auto'), free weight-0 models meter
// against the high `freeRpm` bucket — so "switch to a free model to
// avoid rate limits" actually works. Paid models keep the low `rpm`.
- // The two buckets are independent; the daily cost cap below is the
- // real backstop against runaway free loops.
+ // The two buckets are independent; Cloudflare's spend policy is the
+ // backstop against runaway free loops.
const rateLimit = await checkRateLimit(request, env, authResult, {
freeModel: isFreeModel(body.model),
});
@@ -640,62 +555,8 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
return rateLimit.response;
}
- const cloudflareGateway = shouldUseHostedChatGateway(env, body.model);
- let legacyRescueFallback = false;
- // Legacy mode retains the paid weighted-query admission gate. In
- // Cloudflare mode the provider-cost spend rules are authoritative for
- // this endpoint; Free's separate two-message lease remains above.
- let usage: Awaited<ReturnType<typeof trackUsage>> | null = null;
- if (!cloudflareGateway) {
- const ipAddress = request.headers.get('cf-connecting-ip') || undefined;
- usage = await trackUsage(env, authResult.deviceId, usageTier, authResult.userId, ipAddress, body.model);
- }
- if (usage && !usage.allowed) {
- console.warn('hosted AI admission rejected', {
- gate: 'daily_query',
- tier: authResult.tier,
- accountPlan: authResult.accountPlan,
- });
- const creditsExhausted = (usage.creditsRemaining ?? 0) <= 0;
- const allowanceError = {
- status: 429,
- code: creditsExhausted ? 'credits_exhausted' : 'daily_limit_exceeded',
- };
- const rescueFallbackBody = resolveBackgroundFallbackBody({
- enabled: isBackgroundRequest(request) && hasPaidHostedAiPlan(authResult),
- error: allowanceError,
- body,
- env,
- });
- if (rescueFallbackBody) {
- legacyRescueFallback = true;
- body = rescueFallbackBody;
- } else return addCorsHeaders(createErrorResponse(429, JSON.stringify({
- ...buildDailyUsageLimitError(
- usage,
- usageTier,
- authResult.accountPlan,
- creditsExhausted
- ? `You've used all free queries and have no credits remaining. Buy more at screenpi.pe`
- : `You've used all ${usage.limit} free AI queries for today. Resets at ${usage.resetsAt}`,
- ),
- upgrade_options: {
- buy_credits: {
- url: 'https://screenpi.pe/onboarding',
- benefit: 'Credits extend your daily limit — use anytime',
- },
- subscribe: {
- url: 'https://screenpi.pe/onboarding',
- benefit: 'Frontier Claude and GPT models, higher AI limits, and encrypted sync',
- price: '$29/mo',
- },
- },
- })));
- }
-
- // Reserve the Free-preview allowance only after every other request gate.
- // Its legacy fail-open behavior is still bounded by the fail-closed shared
- // spend lease acquired immediately below.
+ // Free preview remains a separate product allowance. Provider-cost spend
+ // policy is enforced by Cloudflare AI Gateway for every hosted model.
let freeChatLease: FreeChatLease | null = null;
if (freeChat.mode === 'metered') {
const reservation = await reserveFreeChatRequest(env, freeChat);
@@ -705,94 +566,26 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
freeChatLease = reservation.lease;
}
- // Serialize priced work within its foreground/background lane. A scheduled
- // pipe must not block a user who is actively waiting in chat.
const latency = resolveLatencyClass(request, body, env);
- const gatewayContext = cloudflareGateway
- ? await buildHostedChatGatewayContext(authResult, body.model, latency)
- : undefined;
- let dailyCostReservation: DailyCostHold | null = null;
- if (!cloudflareGateway && !legacyRescueFallback) {
- const costReservation = await reserveDailyCostCap(
- env,
- authResult.deviceId,
- authResult.tier,
- body.model,
- new Date(),
- isBackgroundRequest(request) ? 'background' : 'interactive',
- costReservationShape(body, rawRequestBytes),
- authResult.accountPlan,
- authResult.hostedAiTrial === true,
- );
- if (!costReservation.allowed) {
- let rejectionReason: string | undefined;
- try {
- const payload = await costReservation.response.clone().json() as { error?: unknown };
- if (typeof payload.error === 'string') {
- rejectionReason = payload.error;
- try {
- const nested = JSON.parse(payload.error) as { error?: unknown };
- if (typeof nested.error === 'string') rejectionReason = nested.error;
- } catch {
- // The error was already a plain code.
- }
- }
- } catch {
- // Preserve the original response even if diagnostic decoding fails.
- }
- console.warn('hosted AI admission rejected', {
- gate: 'cost_reservation',
- reason: rejectionReason,
- tier: authResult.tier,
- accountPlan: authResult.accountPlan,
- hostedAiTrial: authResult.hostedAiTrial === true,
- status: costReservation.response.status,
- });
- const allowanceError = { status: costReservation.response.status, code: rejectionReason };
- const rescueFallbackBody = resolveBackgroundFallbackBody({
- enabled: isBackgroundRequest(request) && hasPaidHostedAiPlan(authResult),
- error: allowanceError,
- body,
- env,
- });
- if (rescueFallbackBody) {
- legacyRescueFallback = true;
- body = rescueFallbackBody;
- } else {
- if (freeChatLease) await releaseFreeChatLease(env, freeChatLease);
- return costReservation.response;
- }
- }
- if (costReservation.allowed && !legacyRescueFallback) {
- dailyCostReservation = costReservation.reservation;
- }
- }
+ const gatewayContext = await buildHostedChatGatewayContext(authResult, body.model, latency);
- // Route latency-tolerant (background) traffic to the cheaper flex tier.
let leaseReleased = false;
const releaseLease = async () => {
if (!freeChatLease || leaseReleased) return;
leaseReleased = true;
await releaseFreeChatLease(env, freeChatLease);
};
- let costSettlement: Promise<boolean>;
const attachLeaseRelease = (outgoing: Response): Response => {
- const costBound = withDailyCostSettlement(
- outgoing,
- env,
- dailyCostReservation,
- costSettlement,
- );
- if (!freeChatLease) return costBound;
- return withFreeChatLeaseRelease(costBound, () => {
+ if (!freeChatLease) return outgoing;
+ return withFreeChatLeaseRelease(outgoing, () => {
const release = releaseLease();
ctx.waitUntil(release);
return release;
});
};
- // Add credit info header if paid via credits. Time it for the cost log
- // (Date.now advances across the upstream fetch I/O) — ≈ TTFB for stream,
+ // Time the request for the cost log (Date.now advances across the upstream
+ // fetch I/O) — approximately TTFB for stream,
// total for non-stream. Includes any router/embed overhead.
const reqStart = Date.now();
try {
@@ -841,7 +634,7 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
served_tier: response.headers.get('x-screenpipe-served-tier'),
router_tier: routerTier,
workload: latency,
- gateway_mode: cloudflareGateway ? 'cloudflare' : 'legacy',
+ gateway_mode: 'cloudflare',
latency_ms: latencyMs,
status_code: response.status,
});
@@ -856,19 +649,16 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
// Cloudflare records blocked attempts with no provider cost. Keep D1 as
// a completed-cost comparison only; estimating a 429 with default token
// counts would manufacture spend that the Gateway correctly reports as $0.
- if (cloudflareGateway && !response.ok) {
- costSettlement = Promise.resolve(true);
// Log cost — for streaming, intercept SSE events to get real token counts.
- } else if (body.stream) {
+ if (response.ok && body.stream) {
const { response: trackedResponse, usage: usagePromise } = trackResponseUsage(response, 'openai');
response = trackedResponse;
- costSettlement = usagePromise.then(u => logCost(env, {
- settlement_id: dailyCostReservation?.key,
+ void usagePromise.then(u => logCost(env, {
device_id: authResult.deviceId,
user_id: authResult.userId,
tier: authResult.tier,
hosted_ai_trial: authResult.hostedAiTrial === true,
- budgeted: !cloudflareGateway,
+ budgeted: false,
provider: inferProvider(servedModel),
model: pricedModel,
input_tokens: u.input_tokens ?? null,
@@ -881,19 +671,16 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
cache_read_tokens: u.cache_read_input_tokens,
cache_creation_tokens: u.cache_creation_input_tokens,
usage_complete: u.usage_complete,
- }, dailyCostReservation?.reservedMicroUsd),
+ }),
endpoint: '/v1/chat/completions',
stream: true,
latency_ms: latencyMs,
router_tier: routerTier,
- lane: dailyCostReservation?.lane,
- cost_ledger_epoch: dailyCostReservation?.ledgerEpoch,
- cost_total_ledger_epoch: dailyCostReservation?.totalLedgerEpoch,
}));
- } else {
- costSettlement = settleActualOrReservedCost(
+ } else if (response.ok) {
+ void settleActualOrReservedCost(
env,
- dailyCostReservation,
+ null,
reservedCostAttribution(
authResult,
pricedModel,
@@ -911,12 +698,11 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
const cacheRead = json?.usage?.prompt_tokens_details?.cached_tokens ?? null;
const cacheCreation = json?.usage?.cache_creation_input_tokens ?? null;
return await logCost(env, {
- settlement_id: dailyCostReservation?.key,
device_id: authResult.deviceId,
user_id: authResult.userId,
tier: authResult.tier,
hosted_ai_trial: authResult.hostedAiTrial === true,
- budgeted: !cloudflareGateway,
+ budgeted: false,
provider: inferProvider(servedModel),
model: pricedModel,
input_tokens: inputTokens,
@@ -926,39 +712,19 @@ export async function handleRequest(request: Request, env: Env, ctx: ExecutionCo
estimated_cost_usd: getNonStreamSettlementCost(pricedModel, inputTokens, outputTokens, {
cache_read_tokens: cacheRead,
cache_creation_tokens: cacheCreation,
- }, dailyCostReservation?.reservedMicroUsd),
+ }),
endpoint: '/v1/chat/completions',
stream: false,
latency_ms: latencyMs,
router_tier: routerTier,
- lane: dailyCostReservation?.lane,
- cost_ledger_epoch: dailyCostReservation?.ledgerEpoch,
- cost_total_ledger_epoch: dailyCostReservation?.totalLedgerEpoch,
});
},
);
}
- if (usage?.paidVia === 'credits' && usage.creditsRemaining !== undefined) {
- const newResponse = new Response(response.body, response);
- newResponse.headers.set('X-Credits-Remaining', String(usage.creditsRemaining));
- newResponse.headers.set('X-Paid-Via', 'credits');
- return attachLeaseRelease(newResponse);
- }
return attachLeaseRelease(response);
} catch (error) {
await releaseLease();
- await settleProviderException(
- env,
- dailyCostReservation,
- reservedCostAttribution(
- authResult,
- body.model,
- '/v1/chat/completions',
- body.stream === true,
- { latencyMs: Date.now() - reqStart },
- ),
- );
throw error;
}
}
diff --git a/packages/ai-gateway/src/services/api-audit.ts b/packages/ai-gateway/src/services/api-audit.ts
index b894675d1..8de439038 100644
--- a/packages/ai-gateway/src/services/api-audit.ts
+++ b/packages/ai-gateway/src/services/api-audit.ts
@@ -35,7 +35,7 @@ export interface ApiRouteAudit {
served_tier?: string | null;
router_tier?: string | null;
workload: 'interactive' | 'background';
- gateway_mode: 'legacy' | 'cloudflare';
+ gateway_mode: 'cloudflare';
latency_ms: number;
status_code: number;
}
diff --git a/packages/ai-gateway/src/services/cloudflare-ai-gateway.ts b/packages/ai-gateway/src/services/cloudflare-ai-gateway.ts
index 1d160d4bd..59e65f885 100644
--- a/packages/ai-gateway/src/services/cloudflare-ai-gateway.ts
+++ b/packages/ai-gateway/src/services/cloudflare-ai-gateway.ts
@@ -6,8 +6,6 @@ import type { AuthResult, Env } from '../types';
import { isFrontierModel } from './cost-tracker';
import { getHostedAiPlan } from './hosted-ai-policy';
import { SCREENPIPE_GLM_MODEL } from '../providers/screenpipe-glm';
-
-export type HostedChatGatewayMode = 'legacy' | 'cloudflare';
export type HostedChatPlan = 'free' | 'basic' | 'business' | 'business_max' | 'business_ultra' | 'super_admin' | 'internal';
export type HostedChatLane = 'auto' | 'explicit' | 'frontier';
export type HostedChatRequestLane = Exclude<HostedChatLane, 'frontier'>;
@@ -70,25 +68,6 @@ class HostedChatGatewayConfigurationError extends Error {
}
}
-export function getHostedChatGatewayMode(env: Pick<Env, 'HOSTED_CHAT_GATEWAY_MODE'>): HostedChatGatewayMode {
- return String(env.HOSTED_CHAT_GATEWAY_MODE ?? '').trim().toLowerCase() === 'cloudflare'
- ? 'cloudflare'
- : 'legacy';
-}
-
-export function isHostedChatGatewayEnabled(env: Pick<Env, 'HOSTED_CHAT_GATEWAY_MODE'>): boolean {
- return getHostedChatGatewayMode(env) === 'cloudflare';
-}
-
-/** Keep the global rollout gate for existing providers, but GLM is only
- * exposed through Screenpipe's reviewed custom Gateway provider. */
-export function shouldUseHostedChatGateway(
- env: Pick<Env, 'HOSTED_CHAT_GATEWAY_MODE'>,
- model: string,
-): boolean {
- return isHostedChatGatewayEnabled(env) || model.toLowerCase() === SCREENPIPE_GLM_MODEL;
-}
-
function collapsePlan(auth: AuthResult): HostedChatPlan {
if (auth.service === true) return 'internal';
if (auth.accountPlan === 'enterprise') return 'business_ultra';
diff --git a/packages/ai-gateway/src/test/cloudflare-ai-gateway.unit.test.ts b/packages/ai-gateway/src/test/cloudflare-ai-gateway.unit.test.ts
index 4cc788cfb..0d574fd70 100644
--- a/packages/ai-gateway/src/test/cloudflare-ai-gateway.unit.test.ts
+++ b/packages/ai-gateway/src/test/cloudflare-ai-gateway.unit.test.ts
@@ -10,9 +10,7 @@ import {
cloudflareSpendLimitRuleId,
gatewayProviderForModel,
getHostedChatGatewayConnection,
- getHostedChatGatewayMode,
isCloudflareSpendLimitError,
- shouldUseHostedChatGateway,
withHostedChatLane,
} from '../services/cloudflare-ai-gateway';
@@ -28,14 +26,6 @@ function auth(overrides: Partial<AuthResult> = {}): AuthResult {
}
describe('Cloudflare hosted-chat metadata', () => {
- it('defaults safely to legacy unless cloudflare is explicit', () => {
- expect(getHostedChatGatewayMode({})).toBe('legacy');
- expect(getHostedChatGatewayMode({ HOSTED_CHAT_GATEWAY_MODE: 'legacy' })).toBe('legacy');
- expect(getHostedChatGatewayMode({ HOSTED_CHAT_GATEWAY_MODE: 'CLOUDFLARE' })).toBe('cloudflare');
- expect(shouldUseHostedChatGateway({}, 'gpt-5.6-luna')).toBe(false);
- expect(shouldUseHostedChatGateway({}, 'glm-5.3-flash-reap50-iq3m')).toBe(true);
- });
-
it('hashes the account identity and sends only the five reviewed fields', async () => {
const first = await buildHostedChatGatewayContext(auth(), 'auto', 'interactive');
const second = await buildHostedChatGatewayContext(
@@ -93,7 +83,6 @@ describe('Cloudflare provider-native connection', () => {
it('uses the Workers binding, BYOK, metadata-only logs, and disables Gateway retries', async () => {
const calls: string[] = [];
const env = {
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_AI_GATEWAY_ID: 'screenpipe-staging',
CLOUDFLARE_AI_GATEWAY_TOKEN: 'local-oauth-token',
AI: {
@@ -122,7 +111,6 @@ describe('Cloudflare provider-native connection', () => {
it('uses the explicit provider-native Gateway URL during local development', async () => {
const env = {
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_AI_GATEWAY_ID: 'gateway-staging',
CLOUDFLARE_AI_GATEWAY_BASE_URL:
'https://gateway.ai.cloudflare.com/v1/account-id/gateway-staging/compat/chat/completions',
@@ -139,7 +127,6 @@ describe('Cloudflare provider-native connection', () => {
it('routes Screenpipe GLM through the custom provider with container auth instead of BYOK', async () => {
const calls: string[] = [];
const env = {
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_AI_GATEWAY_ID: 'gateway-staging',
TINFOIL_GLM_API_KEY: 'glm-container-secret',
AI: {
diff --git a/packages/ai-gateway/src/test/free-chat-route.unit.test.ts b/packages/ai-gateway/src/test/free-chat-route.unit.test.ts
index ce750eec4..9f82c34b3 100644
--- a/packages/ai-gateway/src/test/free-chat-route.unit.test.ts
+++ b/packages/ai-gateway/src/test/free-chat-route.unit.test.ts
@@ -264,7 +264,7 @@ describe('/v1/chat/completions free-plan route policy', () => {
cost_limit_reached: boolean;
hosted_ai: {
plan: string;
- included_credits: number;
+ included_credits: number | null;
model_access: string[];
};
};
@@ -272,10 +272,11 @@ describe('/v1/chat/completions free-plan route policy', () => {
expect(response.status).toBe(200);
expect(body).toMatchObject({
tier: 'anonymous',
- cost_limit_reached: false,
+ cost_limit_reached: null,
hosted_ai: {
plan: 'free',
- included_credits: 10,
+ included_credits: null,
+ allowance_managed_by: 'cloudflare',
},
});
expect(body.hosted_ai.model_access).toContain('auto');
@@ -675,7 +676,7 @@ describe('/v1/chat/completions free-plan route policy', () => {
hosted_ai: {
plan: string;
trial: boolean;
- included_credits: number;
+ included_credits: number | null;
model_access: string[];
};
};
@@ -684,13 +685,14 @@ describe('/v1/chat/completions free-plan route policy', () => {
expect(result.hosted_ai).toMatchObject({
plan: 'business',
trial: true,
- included_credits: 400,
+ included_credits: null,
+ allowance_managed_by: 'cloudflare',
});
expect(result.hosted_ai.model_access).toContain('claude-fable-5');
expect(result.hosted_ai.model_access).not.toContain('*');
});
- it('fails the usage endpoint closed when private controls are missing', async () => {
+ it('does not depend on retired private text-cost controls for usage', async () => {
globalThis.fetch = mock(async (input: RequestInfo | URL) => {
const url = String(input);
if (url === 'https://screenpipe.com/api/user') {
@@ -722,8 +724,10 @@ describe('/v1/chat/completions free-plan route policy', () => {
headers: { Authorization: 'Bearer eyJ.missing.private.controls' },
}), missingControlsEnv as unknown as Env, ctx);
- expect(response.status).toBe(503);
- expect(await errorCode(response)).toBe('cost_control_unavailable');
+ expect(response.status).toBe(200);
+ expect(await response.json()).toMatchObject({
+ hosted_ai: { allowance_managed_by: 'cloudflare' },
+ });
});
it.each([
@@ -824,7 +828,6 @@ describe('/v1/chat/completions free-plan route policy', () => {
}) as typeof fetch;
const cloudflareEnv = {
...env,
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_AI_GATEWAY_ID: gatewayId,
CLOUDFLARE_ACCOUNT_ID: '9850df1eb8fd807eb8e06f4057b473f1',
CLOUDFLARE_API_TOKEN: 'read-only-token',
@@ -890,7 +893,7 @@ describe('/v1/chat/completions free-plan route policy', () => {
]);
});
- it('bypasses legacy paid admission and reaches Gateway routing when D1 is unavailable', async () => {
+ it('always reaches Gateway routing when legacy D1 admission is unavailable', async () => {
let gatewayCalls = 0;
const d1Statements: string[] = [];
globalThis.fetch = mock(async (input: RequestInfo | URL) => {
@@ -911,7 +914,6 @@ describe('/v1/chat/completions free-plan route policy', () => {
}) as typeof fetch;
const cloudflareEnv = {
...env,
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_AI_GATEWAY_ID: 'hosted-chat-test',
OPENAI_API_KEY: '',
AI: {
@@ -942,8 +944,8 @@ describe('/v1/chat/completions free-plan route policy', () => {
ctx,
);
- // A legacy admission read would return cost_control_unavailable before
- // provider routing. The intentional 502 proves Gateway resolution ran.
+ // The intentional 502 proves Gateway resolution ran without a mode variable
+ // or a legacy admission read.
expect(response.status).toBe(502);
expect(gatewayCalls).toBeGreaterThan(0);
expect(await response.text()).not.toContain('cost_control_unavailable');
@@ -951,9 +953,9 @@ describe('/v1/chat/completions free-plan route policy', () => {
});
it('returns canonical Max and Ultra capacity from desktop-compatible user responses', async () => {
- for (const [billingPlan, usageTier, dailyLimit] of [
- ['pro_max', 'business_max', 120],
- ['pro_ultra', 'business_ultra', 240],
+ for (const [billingPlan, usageTier, dailyLimit, upgradeEligible] of [
+ ['pro_max', 'business_max', 120, true],
+ ['pro_ultra', 'business_ultra', 240, false],
] as const) {
const clerkId = `user_${billingPlan}`;
verifyTokenMock.mockImplementation(async () => ({ sub: clerkId }) as any);
@@ -980,8 +982,8 @@ describe('/v1/chat/completions free-plan route policy', () => {
limit_today: dailyLimit,
remaining: dailyLimit,
upsell_banner: false,
- upgrade_eligible: false,
- cost_limit_reached: false,
+ upgrade_eligible: upgradeEligible,
+ cost_limit_reached: null,
});
}
});
@@ -1075,7 +1077,6 @@ describe('/v1/chat/completions free-plan route policy', () => {
});
it.each([
- '/v1/chat/completions',
'/v1/web-search',
'/v1/tinfoil/chat/completions',
'/v1/tinfoil/responses',
diff --git a/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts b/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts
index d0e8ce56f..d4c72ed06 100644
--- a/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts
+++ b/packages/ai-gateway/src/test/local-gateway-harness.integration.test.ts
@@ -25,7 +25,11 @@ describe('local AI gateway harness', () => {
expect(usage.status).toBe(200);
expect(await usage.json()).toMatchObject({
tier: 'subscribed',
- hosted_ai: { plan: 'business' },
+ hosted_ai: {
+ plan: 'internal',
+ allowance_managed_by: 'cloudflare',
+ allowances: null,
+ },
});
const completion = await harness.fetch('/chat/completions', {
@@ -61,19 +65,22 @@ describe('local AI gateway harness', () => {
expect(streamBody).toContain('data: [DONE]');
const costState = await harness.readCostState();
- expect(costState.dailyCostUsd).toBeGreaterThan(0);
+ expect(costState.dailyCostUsd).toBe(0);
expect(costState.activeReservations).toBe(0);
expect(costState.aggregatedRequests).toBe(2);
expect(harness.outboundRequests).toHaveLength(2);
expect(
- harness.outboundRequests.every((request) => request.expected && request.url === 'https://api.openai.com/v1/chat/completions'),
+ harness.outboundRequests.every((request) =>
+ request.expected && request.url.endsWith('/screenpipe-local-e2e/openai/chat/completions'),
+ ),
).toBe(true);
harness.assertNoUnexpectedOutboundRequests();
});
- test('returns the real structured daily-limit contract before provider egress', async () => {
+ test('ignores retired D1 text caps and reaches Cloudflare Gateway policy', async () => {
const harness = await startHarness({
+ providerReply: 'gateway policy owns admission',
privateCostControls: {
MAX_DAILY_FREE_TEXT_COST: '1',
MAX_DAILY_BASIC_TEXT_COST: '1',
@@ -91,23 +98,15 @@ describe('local AI gateway harness', () => {
body: JSON.stringify({
model: 'gpt-5.4-mini',
stream: true,
- messages: [{ role: 'user', content: 'must stop before provider' }],
+ messages: [{ role: 'user', content: 'legacy cap must not block provider' }],
max_tokens: 16,
}),
});
- expect(response.status).toBe(429);
- const outer = (await response.json()) as { error?: string };
- const contract = JSON.parse(outer.error ?? '{}');
- expect(contract).toMatchObject({
- error: 'daily_cost_limit_exceeded',
- plan: 'business',
- required_plan: 'business_max',
- upgrade_url: 'https://screenpipe.com/account/billing?target_plan=pro_max&interval=month',
- can_buy_credits: false,
- byok_supported: true,
- });
- expect(harness.outboundRequests).toHaveLength(0);
+ expect(response.status).toBe(200);
+ expect(await response.text()).toContain('gateway policy owns admission');
+ expect(harness.outboundRequests).toHaveLength(1);
+ expect(harness.outboundRequests[0]?.url).toEndWith('/screenpipe-local-e2e/openai/chat/completions');
harness.assertNoUnexpectedOutboundRequests();
});
diff --git a/packages/ai-gateway/src/test/local-gateway-harness.ts b/packages/ai-gateway/src/test/local-gateway-harness.ts
index ce89c9b61..a54b0566b 100644
--- a/packages/ai-gateway/src/test/local-gateway-harness.ts
+++ b/packages/ai-gateway/src/test/local-gateway-harness.ts
@@ -191,12 +191,11 @@ export class LocalGatewayHarness {
MODEL_GATING_ENABLED: 'true',
PIPE_FRONTIER_POLICY: 'reject',
ROUTER_MODE: 'off',
+ CLOUDFLARE_AI_GATEWAY_ID: cloudflareGatewayId,
+ CLOUDFLARE_AI_GATEWAY_BASE_URL: `${cloudflareGatewayRoot}/compat/chat/completions`,
+ CLOUDFLARE_AI_GATEWAY_TOKEN: 'screenpipe-local-e2e-gateway-token',
...(cloudflareSpendRules ? {
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_ACCOUNT_ID: cloudflareAccountId,
- CLOUDFLARE_AI_GATEWAY_ID: cloudflareGatewayId,
- CLOUDFLARE_AI_GATEWAY_BASE_URL: `${cloudflareGatewayRoot}/compat/chat/completions`,
- CLOUDFLARE_AI_GATEWAY_TOKEN: 'screenpipe-local-e2e-gateway-token',
CLOUDFLARE_API_TOKEN: 'screenpipe-local-e2e-read-token',
} : {}),
},
@@ -212,9 +211,7 @@ export class LocalGatewayHarness {
.text()
.catch(() => null);
}
- const directProvider = request.method === 'POST' && request.url === 'https://api.openai.com/v1/chat/completions';
- const gatewayProvider = cloudflareSpendRules &&
- request.method === 'POST' &&
+ const gatewayProvider = request.method === 'POST' &&
request.url === `${cloudflareGatewayRoot}/openai/chat/completions`;
const gatewaySettings = cloudflareSpendRules &&
request.method === 'GET' &&
@@ -222,7 +219,7 @@ export class LocalGatewayHarness {
const gatewayAnalytics = cloudflareSpendRules &&
request.method === 'POST' &&
request.url === 'https://api.cloudflare.com/client/v4/graphql';
- const expected = directProvider || gatewayProvider || gatewaySettings || gatewayAnalytics;
+ const expected = gatewayProvider || gatewaySettings || gatewayAnalytics;
harness.outboundRequests.push({
url: request.url,
method: request.method,
diff --git a/packages/ai-gateway/src/test/openai-models.unit.test.ts b/packages/ai-gateway/src/test/openai-models.unit.test.ts
index 4a8b12048..defdab786 100644
--- a/packages/ai-gateway/src/test/openai-models.unit.test.ts
+++ b/packages/ai-gateway/src/test/openai-models.unit.test.ts
@@ -60,29 +60,28 @@ describe('OpenAI API model catalog', () => {
expect(ids).not.toContain('gpt-5.3-codex');
});
- it('hides OpenAI models until OPENAI_API_KEY is configured', async () => {
+ it('advertises Gateway-backed OpenAI models without a direct provider key', async () => {
const ids = await listedModelIds({ OPENAI_API_KEY: '' });
- expect(ids).not.toContain('gpt-5.6-terra');
- expect(ids).not.toContain('gpt-5.5');
- expect(ids).not.toContain('gpt-5.5-pro');
- expect(ids).not.toContain('gpt-5.4');
- expect(ids).not.toContain('gpt-5.4-pro');
+ expect(ids).toContain('gpt-5.6-terra');
+ expect(ids).toContain('gpt-5.5');
+ expect(ids).toContain('gpt-5.5-pro');
+ expect(ids).toContain('gpt-5.4');
+ expect(ids).toContain('gpt-5.4-pro');
expect(ids).not.toContain('gpt-5.3-codex');
- expect(ids).not.toContain('gpt-5.4-mini');
- expect(ids).not.toContain('gpt-5.4-nano');
+ expect(ids).toContain('gpt-5.4-mini');
+ expect(ids).toContain('gpt-5.4-nano');
});
- it('hides OpenAI models when OPENAI_API_KEY is a placeholder', async () => {
+ it('ignores a placeholder direct key for Gateway-backed OpenAI models', async () => {
const ids = await listedModelIds({ OPENAI_API_KEY: 'placeholder' });
- expect(ids).not.toContain('gpt-5.5');
- expect(ids).not.toContain('gpt-5.4-mini');
+ expect(ids).toContain('gpt-5.5');
+ expect(ids).toContain('gpt-5.4-mini');
});
- it('publishes zero query weights when Cloudflare manages hosted-chat allowance', async () => {
+ it('publishes zero query weights because Cloudflare manages hosted-chat allowance', async () => {
const models = await listedModels({
- HOSTED_CHAT_GATEWAY_MODE: 'cloudflare',
CLOUDFLARE_AI_GATEWAY_ID: 'screenpipe-staging',
});
expect(models.length).toBeGreaterThan(0);
diff --git a/packages/ai-gateway/src/types.ts b/packages/ai-gateway/src/types.ts
index 23c341d81..99ea207ce 100644
--- a/packages/ai-gateway/src/types.ts
+++ b/packages/ai-gateway/src/types.ts
@@ -156,8 +156,6 @@ export interface Env {
AI: Ai;
/** GPT-5.6 prompt caching: `system` (default/kill switch) or `history`. */
GPT56_HISTORY_CACHE_MODE?: string;
- /** Hosted chat rollout switch. Any value other than `cloudflare` is legacy. */
- HOSTED_CHAT_GATEWAY_MODE?: string;
/** Same-account Cloudflare AI Gateway used by the Workers AI binding. */
CLOUDFLARE_AI_GATEWAY_ID?: string;
/** Local-dev only: Gateway root or compat chat URL when remote binding URL resolution is unavailable. */
diff --git a/packages/ai-gateway/wrangler.toml b/packages/ai-gateway/wrangler.toml
index 496f55546..34389c05a 100644
--- a/packages/ai-gateway/wrangler.toml
+++ b/packages/ai-gateway/wrangler.toml
@@ -66,9 +66,7 @@ GEMINI_FLEX_INTERACTIVE = "false"
MODEL_GATING_ENABLED = "true"
PIPE_FRONTIER_POLICY = "reject"
-# Hosted-chat AI Gateway rollout is managed in Worker dashboard variables so a
-# deploy cannot flip production (keep_vars=true above):
-# - HOSTED_CHAT_GATEWAY_MODE: "legacy" (default/rollback) or "cloudflare".
+# Hosted chat always uses Cloudflare AI Gateway. Production provides:
# - CLOUDFLARE_AI_GATEWAY_ID: same-account Gateway with default OpenAI and
# Anthropic BYOK keys. See CLOUDFLARE_AI_GATEWAY_ROLLOUT.md.
# - CLOUDFLARE_ACCOUNT_ID: account that owns the Gateway and Analytics data.