Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): per-key scope/format lockdown (keyRestriction flag) (#3960)
Gergő Móricz75060fa0a8fc
diff --git a/apps/api/src/__tests__/snips/v2/key-restriction.test.ts b/apps/api/src/__tests__/snips/v2/key-restriction.test.tsnew file mode 100644index 000000000..fd68eb099--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/key-restriction.test.ts@@ -0,0 +1,176 @@+import { eq } from "drizzle-orm";+import { idmux, map, scrape, scrapeRaw, scrapeTimeout } from "./lib";+import { createTestIdUrl, describeIf, Identity, TEST_PRODUCTION } from "../lib";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";+import { parseApi } from "../../../lib/parseApi";++// Needs idmux (to create teams with the keyRestriction flag) and direct DB+// access (to seed key_restriction_config), so production-suite only.+describeIf(TEST_PRODUCTION)(+ "Key restriction (keyRestriction team flag)",+ () => {+ const seededKeyIds: number[] = [];++ async function seedRestriction(+ identity: Identity,+ restriction: { allowedFormats?: string[]; allowedEndpoints?: string[] },+ ) {+ const [keyRow] = await db+ .select({ id: schema.api_keys.id })+ .from(schema.api_keys)+ .where(eq(schema.api_keys.key, parseApi(identity.apiKey)))+ .limit(1);+ expect(keyRow).toBeDefined();++ await db.insert(schema.key_restriction_config).values({+ api_key_id: keyRow!.id,+ team_id: identity.teamId,+ allowed_formats: restriction.allowedFormats ?? [],+ allowed_endpoints: restriction.allowedEndpoints ?? [],+ });+ seededKeyIds.push(keyRow!.id);+ }++ afterAll(async () => {+ for (const keyId of seededKeyIds) {+ await db+ .delete(schema.key_restriction_config)+ .where(eq(schema.key_restriction_config.api_key_id, keyId));+ }+ });++ it.concurrent(+ "allows markdown scrapes on a markdown-only key",+ async () => {+ const identity = await idmux({+ name: "key-restriction/markdown-allowed",+ credits: 10000,+ flags: { keyRestriction: true },+ });+ await seedRestriction(identity, { allowedFormats: ["markdown"] });++ // Explicit markdown and the implicit default must both pass.+ const doc = await scrape(+ { url: createTestIdUrl(), formats: ["markdown"] },+ identity,+ );+ expect(doc.markdown).toBeDefined();++ const docDefault = await scrape({ url: createTestIdUrl() }, identity);+ expect(docDefault.markdown).toBeDefined();+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "rejects non-allowed formats on a markdown-only key",+ async () => {+ const identity = await idmux({+ name: "key-restriction/format-blocked",+ credits: 10000,+ flags: { keyRestriction: true },+ });+ await seedRestriction(identity, { allowedFormats: ["markdown"] });++ const response = await scrapeRaw(+ { url: createTestIdUrl(), formats: ["markdown", "rawHtml"] },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain(+ "restricted to the following formats",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects content-returning actions on a format-restricted key",+ async () => {+ const identity = await idmux({+ name: "key-restriction/action-blocked",+ credits: 10000,+ flags: { keyRestriction: true },+ });+ await seedRestriction(identity, { allowedFormats: ["markdown"] });++ const response = await scrapeRaw(+ {+ url: createTestIdUrl(),+ formats: ["markdown"],+ actions: [{ type: "screenshot" }],+ },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain("screenshot");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "enforces the endpoint allowlist",+ async () => {+ const identity = await idmux({+ name: "key-restriction/endpoint",+ credits: 10000,+ flags: { keyRestriction: true },+ });+ await seedRestriction(identity, { allowedEndpoints: ["scrape"] });++ // Allowed endpoint group works...+ await scrape({ url: createTestIdUrl() }, identity);++ // ...anything else is rejected at auth.+ const response = await map({ url: "https://firecrawl.dev" }, identity);+ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain(+ "restricted to the following endpoints",+ );+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "does not restrict a flagged team before it configures a restriction",+ async () => {+ const identity = await idmux({+ name: "key-restriction/no-config",+ credits: 10000,+ flags: { keyRestriction: true },+ });++ const doc = await scrape(+ { url: createTestIdUrl(), formats: ["rawHtml"] },+ identity,+ );+ expect(doc.rawHtml).toBeDefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "does not restrict when the flag is off even if a config exists",+ async () => {+ const identity = await idmux({+ name: "key-restriction/no-flag",+ credits: 10000,+ });+ await seedRestriction(identity, { allowedFormats: ["markdown"] });++ const doc = await scrape(+ { url: createTestIdUrl(), formats: ["rawHtml"] },+ identity,+ );+ expect(doc.rawHtml).toBeDefined();+ },+ scrapeTimeout,+ );+ },+);diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 9a82feb41..fbf0bf7c4 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -17,6 +17,7 @@ import {} from "../lib/keyless";import { isKeylessIpSuspicious } from "../lib/spur";import { checkIpRestriction } from "../lib/ip-restriction";+import { checkKeyEndpointRestriction } from "../lib/key-restriction";import { deleteKey, getValue, setValue } from "../services/redis";import { redlock } from "../services/redlock";import { eq } from "drizzle-orm";@@ -817,6 +818,23 @@ async function supaAuthenticateUser(}}+ if (chunk?.flags?.keyRestriction) {+ // Enforced here rather than in route middleware so every authenticated+ // surface (v0/v1/v2, websocket status) goes through the same gate.+ const endpointCheck = await checkKeyEndpointRestriction(+ req.originalUrl ?? req.url ?? "",+ chunk.api_key_id,+ chunk.flags,+ );+ if (!endpointCheck.allowed) {+ return {+ success: false,+ error: endpointCheck.error,+ status: endpointCheck.status,+ };+ }+ }+const team_endpoint_token = token === config.PREVIEW_TOKEN ? iptoken : teamId;try {diff --git a/apps/api/src/controllers/v0/admin/key-restriction-cache-clear.ts b/apps/api/src/controllers/v0/admin/key-restriction-cache-clear.tsnew file mode 100644index 000000000..a5d49c0ce--- /dev/null+++ b/apps/api/src/controllers/v0/admin/key-restriction-cache-clear.ts@@ -0,0 +1,29 @@+import { Request, Response } from "express";+import { clearKeyRestrictionCache } from "../../../lib/key-restriction";+import { logger } from "../../../lib/logger";++// Called by the dashboard after it writes key_restriction_config so+// restriction edits take effect immediately instead of after the 60s+// cache TTL.+export async function keyRestrictionCacheClearController(+ req: Request,+ res: Response,+) {+ try {+ const api_key_id = req.body?.api_key_id;++ if (typeof api_key_id !== "number" || !Number.isInteger(api_key_id)) {+ return res.status(400).json({ error: "api_key_id is required" });+ }++ await clearKeyRestrictionCache(api_key_id);++ logger.info("Key restriction config cache cleared", { api_key_id });+ res.json({ ok: true });+ } catch (error) {+ logger.error("Error clearing key restriction cache via API route", {+ error,+ });+ res.status(500).json({ error: "Internal server error" });+ }+}diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex c43fb5f3d..2a30a540e 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -26,6 +26,11 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import {crawlGroup,resolveNewGroupBackend,@@ -52,6 +57,19 @@ export async function batchScrapeController(});}+ const keyRestriction = await checkKeyFormatRestriction(+ formatTypesOf(req.body.formats),+ actionTypesOf(req.body.actions),+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex baf3975af..57674d8f4 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -18,6 +18,11 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import {crawlGroup,resolveNewGroupBackend,@@ -43,6 +48,19 @@ export async function crawlController(});}+ const keyRestriction = await checkKeyFormatRestriction(+ formatTypesOf(req.body.scrapeOptions?.formats),+ actionTypesOf(req.body.scrapeOptions?.actions),+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 93c22669f..f17ef0337 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -14,6 +14,11 @@ import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import { includesFormat } from "../../lib/format-utils";import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";import { processJobInternal } from "../../services/worker/scrape-worker";@@ -53,6 +58,19 @@ export async function scrapeController(});}+ const keyRestriction = await checkKeyFormatRestriction(+ formatTypesOf(req.body.formats),+ actionTypesOf(req.body.actions),+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 394ed6a73..018698bb9 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -12,6 +12,11 @@ import { v7 as uuidv7 } from "uuid";import { logSearch, logRequest } from "../../services/logging/log_job";import { search } from "../../search";import { logger as _logger } from "../../lib/logger";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import type { Logger } from "winston";import { ScrapeJobTimeoutError } from "../../lib/error";import { captureExceptionWithZdrCheck } from "../../services/sentry";@@ -122,6 +127,24 @@ export async function searchController(try {req.body = searchRequestSchema.parse(req.body);+ const requestedFormats = formatTypesOf(req.body.scrapeOptions?.formats);+ const keyRestriction = await checkKeyFormatRestriction(+ requestedFormats,+ // Search only scrapes (and only runs actions) when formats are+ // requested; without them scrapeOptions is ignored entirely.+ requestedFormats.length > 0+ ? actionTypesOf(req.body.scrapeOptions?.actions)+ : [],+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+logger = logger.child({version: "v1",query: req.body.query,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 7530169d0..eac175261 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1313,6 +1313,8 @@ export type TeamFlags = {ipWhitelist?: boolean;// gates the per-team API key IP allowlist (ip_restriction_config table)ipRestriction?: boolean;+ // gates the per-key scope/format lockdown (key_restriction_config table)+ keyRestriction?: boolean;skipCountryCheck?: boolean;browserBeta?: boolean;bypassCreditChecks?: boolean;diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 599908dc8..ad9df8c80 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -26,6 +26,11 @@ import { logger as _logger } from "../../lib/logger";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { checkPermissions } from "../../lib/permissions";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import {crawlGroup,resolveNewGroupBackend,@@ -53,6 +58,19 @@ export async function batchScrapeController(});}+ const keyRestriction = await checkKeyFormatRestriction(+ formatTypesOf(req.body.formats),+ actionTypesOf(req.body.actions),+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" ||(req.body.zeroDataRetention ?? false);diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex cf9a70820..0568e9705 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -19,6 +19,11 @@ import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";import {crawlGroup,@@ -45,6 +50,19 @@ export async function crawlController(});}+ const keyRestriction = await checkKeyFormatRestriction(+ formatTypesOf(req.body.scrapeOptions?.formats),+ actionTypesOf(req.body.scrapeOptions?.actions),+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex d779a9005..44823a2f7 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -14,6 +14,11 @@ import { hasFormatOfType } from "../../lib/format-utils";import { TransportableError } from "../../lib/error";import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";import { withSpan, setSpanAttributes, SpanKind } from "../../lib/otel-tracer";import { processJobInternal } from "../../services/worker/scrape-worker";import { ScrapeJobData } from "../../types";@@ -91,6 +96,23 @@ export async function scrapeController(});}+ const keyRestriction = await checkKeyFormatRestriction(+ formatTypesOf(req.body.formats),+ actionTypesOf(req.body.actions),+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ setSpanAttributes(span, {+ "scrape.error": keyRestriction.error,+ "scrape.status_code": keyRestriction.status,+ });+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" ||(req.body.zeroDataRetention ?? false) ||diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 05331a269..7d2cdb43b 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -28,6 +28,11 @@ import type { BillingMetadata } from "../../services/billing/types";import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import {+ actionTypesOf,+ checkKeyFormatRestriction,+ formatTypesOf,+} from "../../lib/key-restriction";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -61,6 +66,24 @@ export async function searchController(try {req.body = searchRequestSchema.parse(req.body);+ const requestedFormats = formatTypesOf(req.body.scrapeOptions?.formats);+ const keyRestriction = await checkKeyFormatRestriction(+ requestedFormats,+ // Search only scrapes (and only runs actions) when formats are+ // requested; without them scrapeOptions is ignored entirely.+ requestedFormats.length > 0+ ? actionTypesOf(req.body.scrapeOptions?.actions)+ : [],+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!keyRestriction.allowed) {+ return res.status(keyRestriction.status).json({+ success: false,+ error: keyRestriction.error,+ });+ }+if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 782decfe0..61ee4ad23 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1545,6 +1545,8 @@ export type TeamFlags = {ipWhitelist?: boolean;// gates the per-team API key IP allowlist (ip_restriction_config table)ipRestriction?: boolean;+ // gates the per-key scope/format lockdown (key_restriction_config table)+ keyRestriction?: boolean;bypassCreditChecks?: boolean;debugBranding?: boolean;maxBrowserSessions?: number;diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 545e2926b..8fbdc3c4c 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -13,6 +13,9 @@ import {timestamp,date,bytea,+ check,+ foreignKey,+ unique,} from "drizzle-orm/pg-core";const ts = (name: string) =>@@ -60,15 +63,23 @@ export const agents = pgTable("agents", {error: text("error"),});-export const api_keys = pgTable("api_keys", {- id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),- created_at: ts("created_at").defaultNow(),- key: uuid("key").defaultRandom(),- name: text("name"),- team_id: uuid("team_id"),- owner_id: uuid("owner_id"),- agent_provisioned: boolean("agent_provisioned").default(false),-});+export const api_keys = pgTable(+ "api_keys",+ {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ created_at: ts("created_at").defaultNow(),+ key: uuid("key").defaultRandom(),+ name: text("name"),+ team_id: uuid("team_id"),+ owner_id: uuid("owner_id"),+ agent_provisioned: boolean("agent_provisioned").default(false),+ },+ table => [+ // Target of key_restriction_config's composite FK, which pins a+ // restriction row's team_id to the key's actual team.+ unique("api_keys_id_team_id_key").on(table.id, table.team_id),+ ],+);export const batch_scrapes = pgTable("batch_scrapes", {id: uuid("id").notNull(),@@ -254,6 +265,44 @@ export const ip_restriction_config = pgTable("ip_restriction_config", {updated_at: ts("updated_at").notNull().defaultNow(),});+// Per-API-key scope/format lockdown, gated by the keyRestriction team flag.+// Each allowlist is enforced only when non-empty; allowed_formats holds v2+// format type names, allowed_endpoints holds endpoint group names.+export const key_restriction_config = pgTable(+ "key_restriction_config",+ {+ id: uuid("id").notNull().defaultRandom(),+ api_key_id: bigintNum("api_key_id").notNull().unique(),+ team_id: uuid("team_id").notNull(),+ allowed_formats: jsonb("allowed_formats")+ .notNull()+ .default(sql`'[]'::jsonb`),+ allowed_endpoints: jsonb("allowed_endpoints")+ .notNull()+ .default(sql`'[]'::jsonb`),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ },+ table => [+ // Composite FK keeps team_id consistent with the key's actual team, so+ // team-scoped dashboard listings can't target another team's key.+ foreignKey({+ columns: [table.api_key_id, table.team_id],+ foreignColumns: [api_keys.id, api_keys.team_id],+ }).onDelete("cascade"),+ // The API treats non-array/non-string junk as an empty allowlist, which+ // would silently lift the restriction — make such rows unrepresentable.+ check(+ "allowed_formats_is_string_array",+ sql`jsonb_typeof(${table.allowed_formats}) = 'array' AND NOT jsonb_path_exists(${table.allowed_formats}, '$[*] ? (@.type() != "string")')`,+ ),+ check(+ "allowed_endpoints_is_string_array",+ sql`jsonb_typeof(${table.allowed_endpoints}) = 'array' AND NOT jsonb_path_exists(${table.allowed_endpoints}, '$[*] ? (@.type() != "string")')`,+ ),+ ],+);+export const llm_texts = pgTable("llm_texts", {id: uuid("id").notNull().defaultRandom(),origin_url: text("origin_url").notNull(),diff --git a/apps/api/src/lib/key-restriction.test.ts b/apps/api/src/lib/key-restriction.test.tsnew file mode 100644index 000000000..2595cdcfb--- /dev/null+++ b/apps/api/src/lib/key-restriction.test.ts@@ -0,0 +1,283 @@+import {+ actionTypesOf,+ areFormatsAllowed,+ classifyEndpoint,+ formatTypesOf,+ isEndpointAllowed,+ normalizeFormatForRestriction,+ type KeyRestrictionConfig,+} from "./key-restriction";++const unrestricted: KeyRestrictionConfig = {+ allowedFormats: [],+ allowedEndpoints: [],+};++function config(+ overrides: Partial<KeyRestrictionConfig>,+): KeyRestrictionConfig {+ return { ...unrestricted, ...overrides };+}++describe("classifyEndpoint", () => {+ it("classifies core scrape-output endpoints for v1 and v2", () => {+ expect(classifyEndpoint("/v2/scrape")).toEqual({+ api: "v2",+ group: "scrape",+ alwaysAllowed: false,+ });+ expect(classifyEndpoint("/v1/scrape")).toEqual({+ api: "v1",+ group: "scrape",+ alwaysAllowed: false,+ });+ expect(classifyEndpoint("/v2/crawl")).toMatchObject({ group: "crawl" });+ expect(classifyEndpoint("/v2/map")).toMatchObject({ group: "map" });+ expect(classifyEndpoint("/v2/search")).toMatchObject({ group: "search" });+ expect(classifyEndpoint("/v2/extract")).toMatchObject({+ group: "extract",+ });+ expect(classifyEndpoint("/v2/agent")).toMatchObject({ group: "agent" });+ expect(classifyEndpoint("/v2/parse")).toMatchObject({ group: "parse" });+ });++ it("groups job status/cancel endpoints with their job type", () => {+ expect(classifyEndpoint("/v2/scrape/abc-123")).toMatchObject({+ group: "scrape",+ });+ expect(classifyEndpoint("/v2/crawl/abc-123")).toMatchObject({+ group: "crawl",+ });+ expect(classifyEndpoint("/v2/crawl/abc-123/errors")).toMatchObject({+ group: "crawl",+ });+ expect(classifyEndpoint("/v2/crawl/ongoing")).toMatchObject({+ group: "crawl",+ });+ expect(classifyEndpoint("/v2/batch/scrape")).toMatchObject({+ group: "batch-scrape",+ });+ expect(classifyEndpoint("/v2/batch/scrape/abc-123")).toMatchObject({+ group: "batch-scrape",+ });+ });++ it("groups interactive-browser surfaces under browser", () => {+ expect(classifyEndpoint("/v2/scrape/abc-123/interact")).toMatchObject({+ group: "browser",+ });+ expect(classifyEndpoint("/v2/browser")).toMatchObject({+ group: "browser",+ });+ expect(classifyEndpoint("/v2/interact/abc-123")).toMatchObject({+ group: "browser",+ });+ });++ it("distinguishes research from search despite the nested path", () => {+ expect(classifyEndpoint("/v2/search/research/papers")).toMatchObject({+ group: "research",+ });+ expect(classifyEndpoint("/v2/research/foo")).toMatchObject({+ group: "research",+ });+ expect(classifyEndpoint("/v2/search")).toMatchObject({ group: "search" });+ });++ it("marks account/metadata endpoints as always allowed", () => {+ expect(classifyEndpoint("/v2/team/credit-usage")).toMatchObject({+ alwaysAllowed: true,+ });+ expect(classifyEndpoint("/v2/concurrency-check")).toMatchObject({+ alwaysAllowed: true,+ });+ expect(classifyEndpoint("/v1/team/queue-status")).toMatchObject({+ alwaysAllowed: true,+ });+ });++ it("classifies v0 and non-API paths", () => {+ expect(classifyEndpoint("/v0/scrape")).toEqual({ api: "v0" });+ expect(classifyEndpoint("/admin/xyz/redis-health")).toBeNull();+ expect(classifyEndpoint("/is-production")).toBeNull();+ });++ it("ignores query strings and returns null group for unknown v2 paths", () => {+ expect(classifyEndpoint("/v2/scrape?foo=bar")).toMatchObject({+ group: "scrape",+ });+ expect(classifyEndpoint("/v2/some-new-endpoint")).toEqual({+ api: "v2",+ group: null,+ alwaysAllowed: false,+ });+ });++ it("does not prefix-match partial segments", () => {+ expect(classifyEndpoint("/v2/scrapefoo")).toMatchObject({ group: null });+ expect(classifyEndpoint("/v2/mapper")).toMatchObject({ group: null });+ });+});++describe("isEndpointAllowed", () => {+ it("allows everything for an unrestricted config", () => {+ expect(isEndpointAllowed("/v0/scrape", unrestricted).allowed).toBe(true);+ expect(isEndpointAllowed("/v2/agent", unrestricted).allowed).toBe(true);+ });++ it("enforces the endpoint allowlist", () => {+ const c = config({ allowedEndpoints: ["scrape", "crawl"] });+ expect(isEndpointAllowed("/v2/scrape", c).allowed).toBe(true);+ expect(isEndpointAllowed("/v2/crawl/abc/errors", c).allowed).toBe(true);+ expect(isEndpointAllowed("/v2/agent", c)).toMatchObject({+ allowed: false,+ status: 403,+ });+ expect(isEndpointAllowed("/v2/search", c).allowed).toBe(false);+ });++ it("always allows account/metadata endpoints for restricted keys", () => {+ const c = config({ allowedEndpoints: ["scrape"] });+ expect(isEndpointAllowed("/v2/team/credit-usage", c).allowed).toBe(true);+ expect(isEndpointAllowed("/v2/concurrency-check", c).allowed).toBe(true);+ });++ it("denies unknown v2 endpoints when an allowlist is set", () => {+ const c = config({ allowedEndpoints: ["scrape"] });+ expect(isEndpointAllowed("/v2/some-new-endpoint", c).allowed).toBe(false);+ });++ it("denies the v0 legacy API to any restricted key", () => {+ const formatsOnly = config({ allowedFormats: ["markdown"] });+ const v0 = isEndpointAllowed("/v0/scrape", formatsOnly);+ expect(v0).toMatchObject({ allowed: false, status: 403 });+ if (!v0.allowed) {+ expect(v0.error).toContain("v0");+ }++ const endpointsOnly = config({ allowedEndpoints: ["scrape"] });+ expect(isEndpointAllowed("/v0/scrape", endpointsOnly).allowed).toBe(false);+ });++ it("does not gate endpoints when only formats are restricted", () => {+ const c = config({ allowedFormats: ["markdown"] });+ expect(isEndpointAllowed("/v2/agent", c).allowed).toBe(true);+ expect(isEndpointAllowed("/v2/scrape", c).allowed).toBe(true);+ });++ it("ignores non-API paths", () => {+ const c = config({ allowedEndpoints: ["scrape"] });+ expect(isEndpointAllowed("/admin/xyz/redis-health", c).allowed).toBe(true);+ });+});++describe("areFormatsAllowed", () => {+ it("allows everything when no format restriction is set", () => {+ expect(areFormatsAllowed(["rawHtml"], [], unrestricted).allowed).toBe(true);+ expect(+ areFormatsAllowed([], ["executeJavascript"], unrestricted).allowed,+ ).toBe(true);+ });++ it("allows requests within the format allowlist", () => {+ const c = config({ allowedFormats: ["markdown"] });+ expect(areFormatsAllowed(["markdown"], [], c).allowed).toBe(true);+ expect(areFormatsAllowed([], [], c).allowed).toBe(true);+ });++ it("rejects formats outside the allowlist", () => {+ const c = config({ allowedFormats: ["markdown"] });+ const result = areFormatsAllowed(["markdown", "rawHtml"], [], c);+ expect(result).toMatchObject({ allowed: false, status: 403 });+ if (!result.allowed) {+ expect(result.error).toContain("rawHtml");+ expect(result.error).toContain("markdown");+ }+ expect(areFormatsAllowed(["html"], [], c).allowed).toBe(false);+ expect(areFormatsAllowed(["screenshot"], [], c).allowed).toBe(false);+ expect(areFormatsAllowed(["changeTracking"], [], c).allowed).toBe(false);+ });++ it("normalizes v1 format aliases before checking", () => {+ const md = config({ allowedFormats: ["markdown"] });+ expect(areFormatsAllowed(["screenshot@fullPage"], [], md).allowed).toBe(+ false,+ );+ expect(areFormatsAllowed(["extract"], [], md).allowed).toBe(false);++ const withJson = config({ allowedFormats: ["markdown", "json"] });+ expect(areFormatsAllowed(["extract"], [], withJson).allowed).toBe(true);++ const withScreenshot = config({+ allowedFormats: ["markdown", "screenshot"],+ });+ expect(+ areFormatsAllowed(["screenshot@fullPage"], [], withScreenshot).allowed,+ ).toBe(true);+ });++ it("rejects content-returning actions on format-restricted keys", () => {+ const c = config({ allowedFormats: ["markdown"] });+ for (const action of ["scrape", "executeJavascript", "pdf"]) {+ const result = areFormatsAllowed(["markdown"], [action], c);+ expect(result).toMatchObject({ allowed: false, status: 403 });+ if (!result.allowed) {+ expect(result.error).toContain(action);+ }+ }+ });++ it("gates the screenshot action on the screenshot format", () => {+ const md = config({ allowedFormats: ["markdown"] });+ expect(areFormatsAllowed(["markdown"], ["screenshot"], md).allowed).toBe(+ false,+ );++ const withScreenshot = config({+ allowedFormats: ["markdown", "screenshot"],+ });+ expect(+ areFormatsAllowed(["markdown"], ["screenshot"], withScreenshot).allowed,+ ).toBe(true);+ });++ it("allows interaction-only actions", () => {+ const c = config({ allowedFormats: ["markdown"] });+ expect(+ areFormatsAllowed(+ ["markdown"],+ ["wait", "click", "write", "press", "scroll"],+ c,+ ).allowed,+ ).toBe(true);+ });+});++describe("format/action extraction helpers", () => {+ it("handles both v1 string and v2 object formats", () => {+ expect(formatTypesOf(["markdown", "rawHtml"])).toEqual([+ "markdown",+ "rawHtml",+ ]);+ expect(+ formatTypesOf([{ type: "markdown" }, { type: "screenshot" }]),+ ).toEqual(["markdown", "screenshot"]);+ expect(formatTypesOf(undefined)).toEqual([]);+ });++ it("extracts action types", () => {+ expect(actionTypesOf([{ type: "wait" }, { type: "screenshot" }])).toEqual([+ "wait",+ "screenshot",+ ]);+ expect(actionTypesOf(undefined)).toEqual([]);+ });++ it("normalizes known aliases and passes through the rest", () => {+ expect(normalizeFormatForRestriction("screenshot@fullPage")).toBe(+ "screenshot",+ );+ expect(normalizeFormatForRestriction("extract")).toBe("json");+ expect(normalizeFormatForRestriction("markdown")).toBe("markdown");+ });+});diff --git a/apps/api/src/lib/key-restriction.ts b/apps/api/src/lib/key-restriction.tsnew file mode 100644index 000000000..da60669a8--- /dev/null+++ b/apps/api/src/lib/key-restriction.ts@@ -0,0 +1,370 @@+import { eq } from "drizzle-orm";+import { dbRr } from "../db/connection";+import * as schema from "../db/schema";+import { deleteKey, getValue, setValue } from "../services/redis";+import { logger } from "./logger";+import type { TeamFlags } from "../controllers/v1/types";++// Propagation delay for dashboard edits to key_restriction_config.+const CONFIG_CACHE_TTL_SECONDS = 60;++const MANAGE_RESTRICTIONS_URL =+ "https://www.firecrawl.dev/app/settings?tab=advanced";++const configCacheKey = (apiKeyId: number) => `key-restriction:${apiKeyId}`;++export type KeyRestrictionConfig = {+ allowedFormats: string[];+ allowedEndpoints: string[];+};++function isKeyRestricted(config: KeyRestrictionConfig): boolean {+ return config.allowedFormats.length > 0 || config.allowedEndpoints.length > 0;+}++// Invalidates the cached config so dashboard edits apply immediately+// (admin route key-restriction-cache-clear).+export async function clearKeyRestrictionCache(+ apiKeyId: number,+): Promise<void> {+ await deleteKey(configCacheKey(apiKeyId));+}++function isStringArray(value: unknown): value is string[] {+ return Array.isArray(value) && value.every(x => typeof x === "string");+}++function asStringArray(value: unknown): string[] {+ return Array.isArray(value)+ ? value.filter((x): x is string => typeof x === "string")+ : [];+}++async function getKeyRestrictionConfig(+ apiKeyId: number,+): Promise<KeyRestrictionConfig> {+ const cacheKey = configCacheKey(apiKeyId);++ try {+ const cached = await getValue(cacheKey);+ if (cached !== null) {+ const parsed = JSON.parse(cached);+ // A corrupted-but-parseable cache entry must not reach enforcement:+ // coercing a bad shape to [] would silently lift the restriction, so+ // anything that isn't strictly two string arrays is a cache miss and+ // the DB-backed config applies instead.+ if (+ parsed &&+ typeof parsed === "object" &&+ !Array.isArray(parsed) &&+ isStringArray(parsed.allowedFormats) &&+ isStringArray(parsed.allowedEndpoints)+ ) {+ return {+ allowedFormats: parsed.allowedFormats,+ allowedEndpoints: parsed.allowedEndpoints,+ };+ }+ logger.warn("Ignoring malformed key restriction cache entry", {+ apiKeyId,+ });+ }+ } catch (error) {+ logger.warn("Failed to read key restriction config cache", {+ apiKeyId,+ error,+ });+ }++ const [row] = await dbRr+ .select({+ allowed_formats: schema.key_restriction_config.allowed_formats,+ allowed_endpoints: schema.key_restriction_config.allowed_endpoints,+ })+ .from(schema.key_restriction_config)+ .where(eq(schema.key_restriction_config.api_key_id, apiKeyId))+ .limit(1);++ // A missing row means the key is unrestricted; cached the same way as a+ // configured row so unrestricted keys of flagged teams also skip the DB.+ const config: KeyRestrictionConfig = {+ allowedFormats: asStringArray(row?.allowed_formats),+ allowedEndpoints: asStringArray(row?.allowed_endpoints),+ };++ try {+ await setValue(cacheKey, JSON.stringify(config), CONFIG_CACHE_TTL_SECONDS);+ } catch (error) {+ logger.warn("Failed to cache key restriction config", { apiKeyId, error });+ }++ return config;+}++// Endpoint groups a key can be allowlisted for. Job-status/cancel endpoints+// share the group of their job type, so allowing e.g. "crawl" covers both+// starting crawls and reading their results. First segment-prefix match wins,+// so nested paths ("/search/research") must precede their parent ("/search").+const ENDPOINT_GROUPS: [string[], string][] = [+ [["scrape", ":jobId", "interact"], "browser"],+ [["scrape"], "scrape"],+ [["batch", "scrape"], "batch-scrape"],+ [["crawl"], "crawl"],+ [["map"], "map"],+ [["search", "research"], "research"],+ [["search"], "search"],+ [["extract"], "extract"],+ [["agent"], "agent"],+ [["parse"], "parse"],+ [["browser"], "browser"],+ [["interact"], "browser"],+ [["monitor"], "monitor"],+ [["research"], "research"],+ [["llmstxt"], "llmstxt"],+ [["deep-research"], "deep-research"],+ [["fireclaw"], "fireclaw"],+];++// Account/metadata endpoints that never fetch web content; always reachable+// so restricted keys keep working with SDK bookkeeping calls.+const ALWAYS_ALLOWED_PREFIXES = [+ "team",+ "concurrency-check",+ "feedback",+ "slack",+ "support",+ "keyless",+];++function matchSegments(segments: string[], pattern: string[]): boolean {+ if (segments.length < pattern.length) return false;+ return pattern.every((part, i) =>+ part === ":jobId" ? segments[i].length > 0 : segments[i] === part,+ );+}++type EndpointClassification =+ | { api: "v0" }+ | { api: "v1" | "v2"; group: string | null; alwaysAllowed: boolean }+ | null;++// Classifies a request path into an allowlistable endpoint group. Returns+// null for paths outside the versioned API surface (health, admin, ...).+export function classifyEndpoint(rawUrl: string): EndpointClassification {+ const pathname = rawUrl.split("?")[0];+ const segments = pathname.split("/").filter(s => s.length > 0);+ const version = segments[0];+ if (version === "v0") {+ return { api: "v0" };+ }+ if (version !== "v1" && version !== "v2") {+ return null;+ }++ const rest = segments.slice(1);+ if (rest.length > 0 && ALWAYS_ALLOWED_PREFIXES.includes(rest[0])) {+ return { api: version, group: null, alwaysAllowed: true };+ }++ for (const [pattern, group] of ENDPOINT_GROUPS) {+ if (matchSegments(rest, pattern)) {+ return { api: version, group, alwaysAllowed: false };+ }+ }++ return { api: version, group: null, alwaysAllowed: false };+}++type KeyRestrictionResult =+ | { allowed: true }+ | { allowed: false; error: string; status: number };++// Pure allowlist decision, split out for unit testing.+export function isEndpointAllowed(+ rawUrl: string,+ config: KeyRestrictionConfig,+): KeyRestrictionResult {+ if (!isKeyRestricted(config)) {+ return { allowed: true };+ }++ const classification = classifyEndpoint(rawUrl);++ if (classification === null) {+ // Not a versioned API path; nothing to enforce here.+ return { allowed: true };+ }++ // The v0 legacy API predates format controls entirely (e.g. raw HTML via+ // pageOptions), so any restricted key loses access to it outright.+ if (classification.api === "v0") {+ return {+ allowed: false,+ error: `Request blocked: this API key is restricted and cannot use the legacy v0 API. Please use the v2 API instead. Team admins can manage key restrictions at ${MANAGE_RESTRICTIONS_URL}`,+ status: 403,+ };+ }++ if (+ config.allowedEndpoints.length === 0 ||+ classification.alwaysAllowed ||+ (classification.group !== null &&+ config.allowedEndpoints.includes(classification.group))+ ) {+ return { allowed: true };+ }++ return {+ allowed: false,+ error: `Request blocked: this API key is restricted to the following endpoints: ${config.allowedEndpoints.join(", ")}. Team admins can manage key restrictions at ${MANAGE_RESTRICTIONS_URL}`,+ status: 403,+ };+}++// Content-returning actions are side channels around the format allowlist:+// their output lands in actions.scrapes / javascriptReturns / pdfs /+// screenshots regardless of the requested formats.+const ACTION_FORMAT_EQUIVALENTS: Record<string, string | null> = {+ screenshot: "screenshot",+ // No format equivalent — always rejected for format-restricted keys.+ scrape: null,+ executeJavascript: null,+ pdf: null,+};++// v1 format names that differ from the v2 type names the allowlist stores.+const V1_FORMAT_ALIASES: Record<string, string> = {+ "screenshot@fullPage": "screenshot",+ extract: "json",+};++export function normalizeFormatForRestriction(format: string): string {+ return V1_FORMAT_ALIASES[format] ?? format;+}++// Extraction helpers so controllers can pass formats in either shape:+// v1 uses plain strings, v2 uses { type } objects.+export function formatTypesOf(+ formats: readonly (string | { type: string })[] | undefined,+): string[] {+ return (formats ?? []).map(f => (typeof f === "string" ? f : f.type));+}++export function actionTypesOf(+ actions: readonly { type: string }[] | undefined,+): string[] {+ return (actions ?? []).map(a => a.type);+}++// Pure format decision, split out for unit testing. `formats` are v2 format+// type names (v1 callers normalize first); `actionTypes` are the requested+// action types, if any.+export function areFormatsAllowed(+ formats: string[],+ actionTypes: string[],+ config: KeyRestrictionConfig,+): KeyRestrictionResult {+ if (config.allowedFormats.length === 0) {+ return { allowed: true };+ }++ const blockedFormats = formats+ .map(normalizeFormatForRestriction)+ .filter(format => !config.allowedFormats.includes(format));+ if (blockedFormats.length > 0) {+ return {+ allowed: false,+ error: `Request blocked: this API key is restricted to the following formats: ${config.allowedFormats.join(", ")}. Requested formats not allowed: ${[...new Set(blockedFormats)].join(", ")}. Team admins can manage key restrictions at ${MANAGE_RESTRICTIONS_URL}`,+ status: 403,+ };+ }++ for (const actionType of actionTypes) {+ if (!(actionType in ACTION_FORMAT_EQUIVALENTS)) {+ continue;+ }+ const equivalent = ACTION_FORMAT_EQUIVALENTS[actionType];+ if (equivalent === null || !config.allowedFormats.includes(equivalent)) {+ return {+ allowed: false,+ error: `Request blocked: the "${actionType}" action returns page content and is not available on this format-restricted API key. Team admins can manage key restrictions at ${MANAGE_RESTRICTIONS_URL}`,+ status: 403,+ };+ }+ }++ return { allowed: true };+}++function keyRestrictionApplies(+ flags: TeamFlags,+ apiKeyId: number | undefined,+): apiKeyId is number {+ // Org-level keyRestriction flag gates everything so unflagged teams pay+ // zero cost. Preview/keyless tokens have no api_key_id and are never+ // restricted (they can't be configured in the dashboard).+ return Boolean(flags?.keyRestriction) && Boolean(apiKeyId);+}++function failClosed(apiKeyId: number, error: unknown): KeyRestrictionResult {+ logger.error("Failed to load key restriction config", { apiKeyId, error });+ // Fail closed: the team explicitly opted into key restriction, so an+ // unverifiable request must not slip through.+ return {+ allowed: false,+ error:+ "Internal error while verifying this API key's restrictions. Please try again shortly.",+ status: 500,+ };+}++/**+ * Enforces the per-key endpoint allowlist (key_restriction_config table),+ * gated by the keyRestriction team flag. Called during authentication so it+ * covers every authenticated route uniformly. An empty or missing config+ * means no restriction, so a team can't lock itself out before configuring.+ */+export async function checkKeyEndpointRestriction(+ rawUrl: string,+ apiKeyId: number | undefined,+ flags: TeamFlags,+): Promise<KeyRestrictionResult> {+ if (!keyRestrictionApplies(flags, apiKeyId)) {+ return { allowed: true };+ }++ let config: KeyRestrictionConfig;+ try {+ config = await getKeyRestrictionConfig(apiKeyId);+ } catch (error) {+ return failClosed(apiKeyId, error);+ }++ return isEndpointAllowed(rawUrl, config);+}++/**+ * Enforces the per-key format allowlist on a parsed scrape-output request.+ * Called by the scrape/batch-scrape/crawl/search controllers after zod+ * parsing, so the checked formats are the normalized ones the job will+ * actually run with — request flags can't sidestep it.+ */+export async function checkKeyFormatRestriction(+ formats: string[],+ actionTypes: string[],+ apiKeyId: number | undefined,+ flags: TeamFlags,+): Promise<KeyRestrictionResult> {+ if (!keyRestrictionApplies(flags, apiKeyId)) {+ return { allowed: true };+ }++ let config: KeyRestrictionConfig;+ try {+ config = await getKeyRestrictionConfig(apiKeyId);+ } catch (error) {+ return failClosed(apiKeyId, error);+ }++ return areFormatsAllowed(formats, actionTypes, config);+}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex b17205b78..c25937ad6 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -5,6 +5,7 @@ import { autumnHealthController } from "../controllers/v0/admin/autumn-health";import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { ipRestrictionCacheClearController } from "../controllers/v0/admin/ip-restriction-cache-clear";+import { keyRestrictionCacheClearController } from "../controllers/v0/admin/key-restriction-cache-clear";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";@@ -46,6 +47,11 @@ if (config.BULL_AUTH_KEY) {wrap(ipRestrictionCacheClearController),);+ adminRouter.post(+ `/admin/${config.BULL_AUTH_KEY}/key-restriction-cache-clear`,+ wrap(keyRestrictionCacheClearController),+ );+adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/feng-check`,wrap(checkFireEngine),