Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

4.9 KB
Raw Blame
import { z } from "zod";
import { handleFirecrawlMap } from "@omniroute/open-sse/handlers/firecrawlMap.ts";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { getProviderCredentialsWithQuotaPreflight } from "@/sse/services/auth";
import { resolveClientApiKey } from "@/sse/services/clientApiKeyResolver";
import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy";
import {
  isKeyScopeBlockedCredentials,
  keyScopeBlockedResponse,
  resolveKeyConnectionScope,
  withoutKeyScopeBlocked,
} from "@/sse/services/keyConnectionScope";
import { isRequireApiKeyEnabled } from "@/shared/utils/featureFlags";
import {
  validateBody,
  isValidationFailure,
  formatValidationMessage,
} from "@/shared/validation/helpers";
import {
  isAllRateLimitedCredentials,
  rateLimitedProviderResponse,
} from "@/app/api/v1/_shared/rateLimit";
import { parseAndValidatePublicUrl } from "@/shared/network/outboundUrlGuard";

const mapSchema = z.object({
  url: z
    .string()
    .url()
    .refine((value) => {
      try {
        parseAndValidatePublicUrl(value);
        return true;
      } catch {
        return false;
      }
    }, "url must be a public http(s) URL"),
  limit: z.number().int().min(1).max(100_000).default(5000),
  search: z.string().max(500).optional(),
  sitemap: z.enum(["skip", "include", "only"]).optional(),
  includeSubdomains: z.boolean().optional(),
  ignoreQueryParameters: z.boolean().optional(),
  ignoreCache: z.boolean().optional(),
});

const CORS_HEADERS = {
  "Access-Control-Allow-Methods": "POST, OPTIONS",
  "Access-Control-Allow-Headers": "*",
};

export async function OPTIONS() {
  return new Response(null, { headers: CORS_HEADERS });
}

export async function POST(request: Request) {
  let rawBody: unknown;
  try {
    rawBody = await request.json();
  } catch {
    return errorResponse(400, "Invalid JSON body");
  }

  const validation = validateBody(mapSchema, rawBody);
  if (isValidationFailure(validation))
    return errorResponse(400, formatValidationMessage(validation.error));

  const { apiKey, valid: apiKeyValid } = await resolveClientApiKey(request);
  if (isRequireApiKeyEnabled() && !apiKey) return errorResponse(401, "Authentication required");
  if (isRequireApiKeyEnabled() && apiKey && !apiKeyValid) {
    return errorResponse(401, "Invalid API key");
  }

  const policy = await enforceApiKeyPolicy(request, "web-fetch");
  if (policy.rejection) return policy.rejection;
  // GHSA-3w2r: Firecrawl connections stay inside the key's connection scope.
  const scope = await resolveKeyConnectionScope(policy.apiKeyInfo);
  if (scope.rejection) return scope.rejection;

  const excludedConnectionIds: string[] = [];
  for (let attempt = 0; attempt < 20; attempt++) {
    const selection = await getProviderCredentialsWithQuotaPreflight(
      "firecrawl",
      null,
      scope.allowedConnections,
      null,
      { excludeConnectionIds: excludedConnectionIds }
    );
    if (isKeyScopeBlockedCredentials(selection) && excludedConnectionIds.length === 0) {
      return keyScopeBlockedResponse("firecrawl", selection);
    }
    const credentials = withoutKeyScopeBlocked(selection);
    if (isAllRateLimitedCredentials(credentials)) {
      return rateLimitedProviderResponse("firecrawl", credentials);
    }
    if (!credentials) {
      return errorResponse(
        excludedConnectionIds.length ? 429 : 400,
        excludedConnectionIds.length
          ? "All configured Firecrawl connections have exhausted credits or are rate limited"
          : "No Firecrawl provider connection configured"
      );
    }

    const selected = credentials as Record<string, unknown>;
    const result = await handleFirecrawlMap(validation.data, {
      apiKey: typeof selected.apiKey === "string" ? selected.apiKey : undefined,
      baseUrl: typeof selected.baseUrl === "string" ? selected.baseUrl : undefined,
      providerSpecificData:
        selected.providerSpecificData && typeof selected.providerSpecificData === "object"
          ? (selected.providerSpecificData as Record<string, unknown>)
          : undefined,
    });
    if ((result.status === 402 || result.status === 429) && typeof selected.id === "string") {
      excludedConnectionIds.push(selected.id);
      continue;
    }
    if (result.status < 200 || result.status >= 300) {
      const upstream = result.data as { error?: unknown };
      const message = typeof upstream?.error === "string" ? upstream.error : "Firecrawl Map failed";
      return errorResponse(result.status, message);
    }
    const payload = result.data as { success?: unknown; links?: unknown } | null;
    if (payload?.success !== true || !Array.isArray(payload.links)) {
      return errorResponse(502, "Firecrawl Map returned an invalid response");
    }
    return Response.json(payload, { headers: CORS_HEADERS });
  }
  return errorResponse(
    429,
    "All configured Firecrawl connections have exhausted credits or are rate limited"
  );
}