Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): give the keyless provider-tools 403 a signup link (#5734)
Nicolasab3acea889c7
diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex 1d2e3fc01..3a6c3c64e 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -9,6 +9,14 @@ const mockKeylessLimitBody = vi.fn().mockResolvedValue({error: "keyless limit reached",reason: "credits",});+const providerToolsRefusal = {+ success: false,+ error: "An API key is required for provider tools.",+ signup_url: "https://firecrawl.dev/k/token",+};+const mockKeylessProviderToolsBody = vi.fn(+ (..._args: any[]) => providerToolsRefusal,+);const mockProjectSearchTotalCredits = vi.fn(() => 0);vi.mock("../../../services/logging/log_job", () => ({@@ -31,6 +39,8 @@ vi.mock("../../../lib/keyless", () => ({KEYLESS_FREE_TIER_LIMIT_MESSAGE: "keyless limit reached",adjustKeylessCredits: vi.fn().mockResolvedValue(undefined),keylessLimitBody: (...args: any[]) => mockKeylessLimitBody(...args),+ keylessProviderToolsBody: (...args: any[]) =>+ mockKeylessProviderToolsBody(...args),logKeylessCreditUsage: vi.fn().mockResolvedValue(undefined),reserveKeylessCredits: (...args: any[]) => mockReserveKeylessCredits(...args),}));@@ -533,6 +543,11 @@ describe("skill discovery source", () => {keyless,);expect(keyless.status).toHaveBeenCalledWith(403);+ expect(keyless.json).toHaveBeenCalledWith(providerToolsRefusal);+ expect(mockKeylessProviderToolsBody).toHaveBeenCalledWith(+ "preview_keyless_203.0.113.8",+ expect.objectContaining({ body: expect.any(Object) }),+ );const zdr = makeRes();await searchController(diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsindex 040790233..489feeffc 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -11,6 +11,7 @@ const mocks = vi.hoisted(() => ({discovery: vi.fn(),recordDiscovered: vi.fn(),markCalls: vi.fn(),+ providerToolsBody: vi.fn(),}));vi.mock("../../lib/alexandria-activity", () => ({markAlexandriaActivity: vi.fn(),@@ -33,6 +34,9 @@ vi.mock("../../services/logging/log_job", () => ({vi.mock("../../lib/key-restriction", () => ({checkKeyFormatRestriction: async () => ({ allowed: true }),}));+vi.mock("../../lib/keyless", () => ({+ keylessProviderToolsBody: mocks.providerToolsBody,+}));vi.mock("../../lib/agent-interop", () => ({isAgentInteropSecretValid: (value: string) => value === "test-secret",}));@@ -42,6 +46,7 @@ vi.mock("../../config", () => ({FIRE_EXCHANGE_URL: "https://x",AGENT_INTEROP_SECRET: "test-secret",USE_DB_AUTHENTICATION: true,+ AGENT_AUTH_RESOURCE_METADATA_URL: "https://x/auth",},}));import { providerScrapeController } from "./scrape-alexandria";@@ -54,11 +59,12 @@ const call = {options: {},};let flags: Record<string, unknown> = { exchangeRetrieve: true };+let teamId = "team";const app = express();app.use(express.json());app.use((req, _res, next) => {Object.assign(req, {- auth: { team_id: "team" },+ auth: { team_id: teamId },acuc: {api_key_id: 12,org_id: "org",@@ -81,6 +87,7 @@ const result = (results: unknown[], executed = true) => ({beforeEach(() => {vi.clearAllMocks();flags = { exchangeRetrieve: true };+ teamId = "team";mocks.log.mockResolvedValue(undefined);mocks.scrapeLog.mockResolvedValue(undefined);mocks.discovery.mockImplementation(async (_teamId, calls: unknown[]) =>@@ -91,6 +98,33 @@ beforeEach(() => {);});+it("refuses a keyless caller with its signup link and the auth discovery header", async () => {+ teamId = "preview_keyless_203.0.113.8";+ const refusal = {+ success: false,+ error: "An API key is required for provider tools.",+ signup_url: "https://firecrawl.dev/k/token",+ };+ mocks.providerToolsBody.mockReturnValue(refusal);+ const response = await request(app)+ .post("/v2/scrape")+ .set("x-origin", "cli")+ .send({ alexandria: call });+ expect(response.status).toBe(403);+ expect(response.body).toEqual(refusal);+ expect(mocks.providerToolsBody).toHaveBeenCalledExactlyOnceWith(+ teamId,+ expect.objectContaining({+ body: { alexandria: call },+ headers: expect.objectContaining({ "x-origin": "cli" }),+ }),+ );+ expect(response.headers["www-authenticate"]).toBe(+ 'Bearer resource_metadata="https://x/auth"',+ );+ expect(mocks.retrieve).not.toHaveBeenCalled();+});+it("returns the Scrape contract, shares identity with the legacy route, and logs once per execution", async () => {const response = await request(app).post("/v2/scrape")diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex c01d78c50..0ea512fe6 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -5,6 +5,8 @@ import { v7 as uuidv7 } from "uuid";import { config } from "../../config";import { logger } from "../../lib/logger";import { isAgentInteropSecretValid } from "../../lib/agent-interop";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { keylessProviderToolsBody } from "../../lib/keyless";import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { billingCountryForTeam } from "../../lib/billing-country";@@ -119,10 +121,10 @@ export async function providerScrapeController(res.setHeader("x-request-id", requestId);if (req.auth.team_id.startsWith("preview_keyless_")) {- return res.status(403).json({- success: false,- error: "An API key is required for provider tools.",- });+ applyAgentAuthDiscoveryHeader(res);+ return res+ .status(403)+ .json(keylessProviderToolsBody(req.auth.team_id, req));}if (!req.acuc)diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 6165998f7..de605455e 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -12,6 +12,7 @@ import { billTeam } from "../../services/billing/credit_billing";import {adjustKeylessCredits,keylessLimitBody,+ keylessProviderToolsBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -182,10 +183,10 @@ async function searchControllerInner((wantsTools || wantsSkills || req.body.domainTools) &&req.auth.team_id.startsWith("preview_keyless_")) {- return res.status(403).json({- success: false,- error: "An API key is required for provider tools.",- });+ applyAgentAuthDiscoveryHeader(res);+ return res+ .status(403)+ .json(keylessProviderToolsBody(req.auth.team_id, req));}if ((wantsTools || wantsSkills) && !req.body.query.trim())return res.status(400).json({diff --git a/apps/api/src/lib/keyless.signup-prompt.test.ts b/apps/api/src/lib/keyless.signup-prompt.test.tsindex 8111080d1..01274f2c7 100644--- a/apps/api/src/lib/keyless.signup-prompt.test.ts+++ b/apps/api/src/lib/keyless.signup-prompt.test.ts@@ -28,6 +28,7 @@ import {KEYLESS_FREE_TIER_LIMIT_MESSAGE,checkKeylessEligibility,keylessLimitBody,+ keylessProviderToolsBody,keylessSignupUrlForIp,keylessTeamId,keylessTeamUuid,@@ -160,6 +161,46 @@ describe("keyless limit prompt", () => {});});+describe("keyless provider tools prompt", () => {+ it("puts the caller's own link in the refusal and reports it", () => {+ const body = keylessProviderToolsBody("preview_keyless_203.0.113.8", {+ body: { origin: "cli" },+ });++ expect(decoded(body.signup_url)).toEqual({+ ipv4: IP,+ surface: "cli",+ reason: "account_only_tool",+ });+ expect(body).toEqual({+ success: false,+ error: expect.stringMatching(+ /^An API key is required for provider tools\. /,+ ),+ signup_url: body.signup_url,+ });+ expect(body.error).toContain(`${body.signup_url}\n`);+ expect(trackKeylessPromptShown).toHaveBeenCalledExactlyOnceWith(+ shown({ surface: "cli", reason: "account_only_tool", httpStatus: 403 }),+ );+ });++ it("gives a World ID caller the regular signup link and reports nothing", () => {+ const body = keylessProviderToolsBody(+ `preview_keyless_wid_${"h".repeat(43)}`,+ {+ body: {},+ },+ );++ expect(body.signup_url).toBe(+ "https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api",+ );+ expect(body.error).toContain(`${body.signup_url}\n`);+ expect(trackKeylessPromptShown).not.toHaveBeenCalled();+ });+});+describe("reportKeylessPromptShown", () => {it("reports an IPv4-mapped IPv6 caller under the IPv4 team id", () => {reportKeylessPromptShown("::ffff:203.0.113.8", "api", "limit", 429, "ref");diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 8755c0d80..a8ff930b7 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -263,6 +263,36 @@ export function keylessLimitPromptForTeam();}+function keylessProviderToolsMessage(signupUrl: string): string {+ return `An API key is required for provider tools. Sign up for a free API key at ${signupUrl}++Then authenticate with:+Authorization: Bearer YOUR_API_KEY`;+}++/**+ * The 403 body for a keyless caller asking for provider tools, with the+ * caller's own signup link. Reports the prompt.+ */+export function keylessProviderToolsBody(+ teamId: string,+ req: Parameters<typeof keylessSignupSurface>[0],+): { success: false; error: string; signup_url: string } {+ const ip = keylessPromptIp(keylessIpFromTeamId(teamId));+ const surface = keylessSignupSurface(req);+ const { url, signupRef } = keylessSignupUrlForIp(+ ip,+ surface,+ "account_only_tool",+ );+ reportKeylessPromptShown(ip, surface, "account_only_tool", 403, signupRef);+ return {+ success: false,+ error: keylessProviderToolsMessage(url),+ signup_url: url,+ };+}+/** Structured response for projected-credit reservation exhaustion. */export async function keylessLimitBody(teamId: string,