Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
/**
* POST /v1/web/fetch
*
* Extract content from a URL using a configured web-fetch provider.
* Supports Firecrawl, Jina Reader, Tavily Extract, and TinyFish Fetch.
*
* Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? }
* Response: { provider, url, content, links, metadata, screenshot_url }
*
* Quota-aware fallback (#8297): when no explicit provider is requested, the
* pool is walked in fixed priority order (fill-first) — a rate-limited or
* quota-exhausted provider is skipped instead of short-circuiting the whole
* request. When an explicit provider is requested, no silent fallback is
* performed — a rate-limited/failing explicit provider surfaces its own error.
*/
import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import {
handleWebFetch,
type WebFetchCredentials,
type WebFetchResult,
WEB_FETCH_PROVIDERS as SHARED_WEB_FETCH_PROVIDERS,
EXPLICIT_ONLY_WEB_FETCH_PROVIDERS,
ANONYMOUS_CAPABLE_WEB_FETCH_PROVIDERS,
type WebFetchProviderId,
} from "@omniroute/open-sse/handlers/webFetch.ts";
import * as log from "@/sse/utils/logger";
import { getProviderCredentialsWithQuotaPreflight } from "@/sse/services/auth";
import { resolveClientApiKey } from "@/sse/services/clientApiKeyResolver";
import { enforceApiKeyPolicyWithScope } from "@/app/api/v1/_shared/keyScopedRoute";
import {
keyAdmitsKeylessProviders,
withoutKeyScopeBlocked,
} from "@/sse/services/keyConnectionScope";
import { isRequireApiKeyEnabled } from "@/shared/utils/featureFlags";
import { v1WebFetchSchema } from "@/shared/validation/schemas";
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
import {
isAllRateLimitedCredentials,
rateLimitedProviderResponse,
type RateLimitedCredentials,
} from "@/app/api/v1/_shared/rateLimit";
const CORS_HEADERS = {
"Access-Control-Allow-Methods": "POST, OPTIONS",
"Access-Control-Allow-Headers": "*",
};
const WEB_FETCH_PROVIDERS = SHARED_WEB_FETCH_PROVIDERS;
// Providers that only understand their own URL shape (context7 takes a library
// reference like context7.com/reactjs/react.dev, not a generic web URL). They
// must be requested explicitly and never win auto-selection for arbitrary URLs.
const EXPLICIT_ONLY_PROVIDERS = EXPLICIT_ONLY_WEB_FETCH_PROVIDERS;
// Providers whose upstream serves an anonymous tier without a key. When no
// connection is configured they resolve to empty credentials instead of a 400.
const ANONYMOUS_CAPABLE_PROVIDERS = ANONYMOUS_CAPABLE_WEB_FETCH_PROVIDERS;
// Providers whose free/low tiers surface quota exhaustion as 402/403 instead
// of (or in addition to) 429. jina-reader has no such quota-status signal —
// a 402/403 there is a real auth/bad-request failure, not exhaustion.
const QUOTA_STATUS_PROVIDERS = new Set<WebFetchProviderId>([
"firecrawl",
"tavily-search",
"tinyfish",
]);
type CredentialsLookup = WebFetchCredentials | RateLimitedCredentials | null;
export async function OPTIONS() {
return new Response(null, { headers: CORS_HEADERS });
}
/**
* Resolve credentials for a web-fetch provider (may be a rate-limited stub,
* real credentials, or null when unconfigured). GHSA-3w2r: bounded by the
* caller key's connection scope (`allowedConnections`, null = unscoped); a
* provider whose connections are all outside it counts as unconfigured.
*/
async function resolveCredentials(
providerId: WebFetchProviderId,
allowedConnections: string[] | null
): Promise<CredentialsLookup> {
try {
const creds = withoutKeyScopeBlocked(
await getProviderCredentialsWithQuotaPreflight(providerId, null, allowedConnections)
);
return (creds as CredentialsLookup) ?? null;
} catch {
return null;
}
}
/** A request-time upstream status that means "try the next provider" instead of giving up. */
function isRetryableWebFetchStatus(providerId: WebFetchProviderId, status?: number): boolean {
if (status === HTTP_STATUS.RATE_LIMITED) return true;
if (status === HTTP_STATUS.PAYMENT_REQUIRED || status === HTTP_STATUS.FORBIDDEN) {
return QUOTA_STATUS_PROVIDERS.has(providerId);
}
return false;
}
/** Find the next untried, non-rate-limited, credentialed provider in pool order. */
async function findNextFallbackProvider(
tried: Set<WebFetchProviderId>,
allowedConnections: string[] | null
): Promise<{ providerId: WebFetchProviderId; credentials: WebFetchCredentials } | null> {
for (const pid of WEB_FETCH_PROVIDERS) {
if (EXPLICIT_ONLY_PROVIDERS.has(pid)) continue;
if (tried.has(pid)) continue;
const creds = await resolveCredentials(pid, allowedConnections);
tried.add(pid);
if (creds && !isAllRateLimitedCredentials(creds)) {
return { providerId: pid, credentials: creds };
}
}
return null;
}
interface WebFetchExecutionInput {
url: string;
format: "markdown" | "html" | "links" | "screenshot";
depth: 0 | 1 | 2;
wait_for_selector?: string;
include_metadata?: boolean;
}
interface WebFetchExecutionResult {
result: WebFetchResult;
provider: WebFetchProviderId;
poolExhausted: boolean;
}
/**
* Execute the web-fetch request. When `allowFallback` is true (auto-select),
* a retryable/quota upstream failure walks the remaining pool in order
* before giving up. Explicit-provider requests never fall back.
*/
async function executeWithFallback(
reqBody: WebFetchExecutionInput,
startProvider: WebFetchProviderId,
startCredentials: WebFetchCredentials,
allowFallback: boolean,
triedProviders: Set<WebFetchProviderId>,
allowedConnections: string[] | null
): Promise<WebFetchExecutionResult> {
let provider = startProvider;
let credentials = startCredentials;
let result = await handleWebFetch(reqBody, credentials, provider);
if (!allowFallback) {
return { result, provider, poolExhausted: false };
}
while (!result.success && isRetryableWebFetchStatus(provider, result.status)) {
const next = await findNextFallbackProvider(triedProviders, allowedConnections);
if (!next) {
return { result, provider, poolExhausted: true };
}
provider = next.providerId;
credentials = next.credentials;
result = await handleWebFetch(reqBody, credentials, provider);
}
return { result, provider, poolExhausted: false };
}
type ResolvedWebFetchTarget =
| {
ok: true;
provider: WebFetchProviderId;
credentials: WebFetchCredentials;
tried: Set<WebFetchProviderId>;
isExplicit: boolean;
}
| { ok: false; response: Response };
/** Resolve credentials for an explicitly requested provider (no fallback allowed). */
async function resolveExplicitTarget(
providerId: WebFetchProviderId,
allowedConnections: string[] | null
): Promise<ResolvedWebFetchTarget> {
const creds = await resolveCredentials(providerId, allowedConnections);
// Anonymous-capable providers never hard-fail on credential problems: the
// anonymous tier does not consume key quota, so a rate-limited (or absent)
// key degrades to an anonymous attempt instead of a 429/400.
// GHSA-3w2r: the anonymous tier sends the URL to a party the key's
// connection allowlist did not approve, so a connection-scoped key uses it
// only when the allowlist names `noauth` (chat's #9057 rule).
if (
ANONYMOUS_CAPABLE_PROVIDERS.has(providerId) &&
(keyAdmitsKeylessProviders(allowedConnections) ||
(creds && !isAllRateLimitedCredentials(creds)))
) {
return {
ok: true,
provider: providerId,
credentials: creds && !isAllRateLimitedCredentials(creds) ? creds : {},
tried: new Set([providerId]),
isExplicit: true,
};
}
if (isAllRateLimitedCredentials(creds)) {
return { ok: false, response: rateLimitedProviderResponse(providerId, creds) };
}
if (!creds) {
return {
ok: false,
response: errorResponse(
HTTP_STATUS.BAD_REQUEST,
`No credentials configured for web-fetch provider: ${providerId}. ` +
`Add an API key for "${providerId}" in the dashboard.`
),
};
}
return {
ok: true,
provider: providerId,
credentials: creds,
tried: new Set([providerId]),
isExplicit: true,
};
}
/**
* Auto-select: walk the pool in fixed priority order (fill-first), skipping
* rate-limited stubs instead of letting them short-circuit the loop (#8297).
*/
async function resolveAutoSelectTarget(
allowedConnections: string[] | null
): Promise<ResolvedWebFetchTarget> {
let firstRateLimited: {
providerId: WebFetchProviderId;
credentials: RateLimitedCredentials;
} | null = null;
for (const pid of WEB_FETCH_PROVIDERS) {
if (EXPLICIT_ONLY_PROVIDERS.has(pid)) continue;
const creds = await resolveCredentials(pid, allowedConnections);
if (isAllRateLimitedCredentials(creds)) {
firstRateLimited ??= { providerId: pid, credentials: creds };
continue;
}
if (creds) {
return {
ok: true,
provider: pid,
credentials: creds,
tried: new Set([pid]),
isExplicit: false,
};
}
}
if (firstRateLimited) {
return {
ok: false,
response: rateLimitedProviderResponse(
firstRateLimited.providerId,
firstRateLimited.credentials
),
};
}
return {
ok: false,
response: errorResponse(
HTTP_STATUS.BAD_REQUEST,
`No credentials configured for any web-fetch provider. ` +
`Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.`
),
};
}
/** Resolve the provider + credentials to use for this request (explicit or auto-select). */
async function resolveWebFetchTarget(
requestedProvider: string | undefined,
allowedConnections: string[] | null
): Promise<ResolvedWebFetchTarget> {
if (requestedProvider) {
return resolveExplicitTarget(requestedProvider as WebFetchProviderId, allowedConnections);
}
return resolveAutoSelectTarget(allowedConnections);
}
export async function POST(request: Request) {
let rawBody: unknown;
try {
rawBody = await request.json();
} catch {
log.warn("WEB_FETCH", "Invalid JSON body");
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
}
const validation = validateBody(v1WebFetchSchema, rawBody);
if (isValidationFailure(validation)) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, validation.error.message);
}
const body = validation.data;
// Optional auth check — when REQUIRE_API_KEY=false, ignore presented
// invalid keys so anonymous access works the same as all other client
// APIs (#7785).
const { apiKey: apiKeyRaw, valid: apiKeyValid } = await resolveClientApiKey(request);
if (isRequireApiKeyEnabled() && !apiKeyRaw) {
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Authentication required");
}
if (isRequireApiKeyEnabled() && apiKeyRaw && !apiKeyValid) {
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
}
// Enforce API key policies. GHSA-3w2r: provider credentials (and the
// fallback walk) stay inside the key's connection scope, as in chat.
const scope = await enforceApiKeyPolicyWithScope(request, "web-fetch");
if (scope.rejection) return scope.rejection;
// Resolve provider + credentials (explicit provider never falls back; #8297)
const target = await resolveWebFetchTarget(body.provider, scope.allowedConnections);
if (!target.ok) return target.response;
log.info("WEB_FETCH", `${target.provider} | ${body.url} | format=${body.format}`);
const {
result,
provider: finalProvider,
poolExhausted,
} = await executeWithFallback(
{
url: body.url,
format: body.format,
depth: body.depth as 0 | 1 | 2,
wait_for_selector: body.wait_for_selector,
include_metadata: body.include_metadata,
},
target.provider,
target.credentials,
!target.isExplicit,
target.tried,
scope.allowedConnections
);
if (poolExhausted) {
return unavailableResponse(
HTTP_STATUS.RATE_LIMITED,
"All configured web-fetch providers are rate limited or quota-exhausted"
);
}
if (!result.success) {
return new Response(
JSON.stringify({
error: { message: result.error ?? "Web fetch failed", type: "web_fetch_error" },
}),
{
status: result.status ?? 502,
headers: { "Content-Type": "application/json", ...CORS_HEADERS },
}
);
}
if (finalProvider !== target.provider) {
log.info("WEB_FETCH", `Fell back from ${target.provider} to ${finalProvider}`);
}
return new Response(JSON.stringify(result.data), {
status: 200,
headers: { "Content-Type": "application/json", ...CORS_HEADERS },
});
}