Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): refuse labs search for US-only teams and relay research refusals as US_ONLY_UNAVAILABLE (#5557)
Gergő Móricz973f7c3535cb
diff --git a/apps/api/src/__tests__/routes/labs.routes.test.ts b/apps/api/src/__tests__/routes/labs.routes.test.tsindex d682a3242..761ab9039 100644--- a/apps/api/src/__tests__/routes/labs.routes.test.ts+++ b/apps/api/src/__tests__/routes/labs.routes.test.ts@@ -19,8 +19,22 @@ vi.mock("../../lib/logger", () => {return { logger };});+const mocks = vi.hoisted(() => ({+ fetch: vi.fn(),+ flags: {} as Record<string, unknown>,+}));++vi.mock("undici", async importOriginal => ({+ ...(await importOriginal<typeof import("undici")>()),+ fetch: mocks.fetch,+}));+vi.mock("../../routes/shared", () => ({- authMiddleware: () => (_req: any, _res: any, next: any) => next(),+ authMiddleware: () => (req: any, _res: any, next: any) => {+ req.auth = { team_id: "team-1" };+ req.acuc = { flags: mocks.flags };+ next();+ },wrap: (fn: any) => fn,}));@@ -96,3 +110,50 @@ describe("labs router", () => {expect(response.body.code).not.toBe("LABS_ROUTE_NOT_PROXIED");});});++describe("labs router for a US-only team", () => {+ beforeEach(() => {+ mocks.fetch.mockReset();+ });++ it.each(LABS_ROUTES)(+ "refuses %s without calling the labs service",+ async route => {+ mocks.flags = { labsSearch: true, usOnly: true };+ const [method, path] = route.split(" ");++ const response = await (request(appWithLabs()) as any)[+ method.toLowerCase()+ ]("/labs" + path.replace(/:\w+/g, "id-1"));++ expect(response.status).toBe(403);+ expect(response.body).toEqual({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error:+ "Labs Search is not available for teams restricted to US-only processing.",+ });+ expect(mocks.fetch).not.toHaveBeenCalled();+ },+ );++ it("still proxies for a team without the US-only restriction", async () => {+ mocks.flags = { labsSearch: true };+ mocks.fetch.mockResolvedValue(+ new Response(JSON.stringify({ success: true, results: [] }), {+ status: 200,+ headers: { "content-type": "application/json" },+ }),+ );++ const response = await request(appWithLabs()).post("/labs/search");++ expect(response.status).toBe(200);+ expect(response.body).toEqual({ success: true, results: [] });+ expect(mocks.fetch).toHaveBeenCalledTimes(1);+ expect(mocks.fetch.mock.calls[0][0]).toBe("https://labs.test/labs/search");+ expect(mocks.fetch.mock.calls[0][1].headers).toMatchObject({+ "x-labs-search-team-id": "team-1",+ });+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-us-only.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-us-only.test.tsnew file mode 100644index 000000000..bacffa169--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-us-only.test.ts@@ -0,0 +1,172 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ fetchResearchUpstream: vi.fn(),+}));++vi.mock("../../../services/logging/log_job", () => ({+ logRequest: vi.fn().mockResolvedValue(undefined),+ logResearchEndpoint: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../lib/research-upstream", async importOriginal => ({+ ...(await importOriginal<typeof import("../../../lib/research-upstream")>()),+ fetchResearchUpstream: mocks.fetchResearchUpstream,+}));++vi.mock("../../../services/billing/credit_billing", () => ({+ billTeam: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../lib/keyless", () => ({+ chargeKeylessCredits: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../lib/index-daily-limit", () => ({+ reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),+ indexDailyLimitRefusal: vi.fn(),+}));++vi.mock("../../../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+}));++import { createDeveloperRouter, createResearchRouter } from "../research-proxy";++const REFUSAL =+ "Paper search is not available for teams restricted to US-only processing.";+const flush = () => new Promise(resolve => setImmediate(resolve));++function upstream(status: number, body: unknown) {+ return {+ ok: status >= 200 && status < 300,+ status,+ headers: new Headers({ "content-type": "application/json" }),+ text: async () => JSON.stringify(body),+ json: async () => body,+ };+}++async function callRoute(+ router: any,+ path: string,+ query: Record<string, unknown>,+ flags: Record<string, unknown>,+) {+ const layer = router.stack.find(+ (candidate: any) =>+ candidate.route?.path === path && candidate.route?.methods?.get,+ );+ const res: any = {+ status: vi.fn(),+ json: vi.fn(),+ send: vi.fn(),+ end: vi.fn(),+ setHeader: vi.fn(),+ };+ res.status.mockReturnValue(res);+ res.json.mockReturnValue(res);+ res.send.mockReturnValue(res);+ layer.route.stack.at(-1).handle(+ {+ method: "GET",+ query,+ params: {},+ body: {},+ headers: {},+ auth: { team_id: "11111111-1111-1111-1111-111111111111" },+ acuc: { api_key_id: 7, flags },+ },+ res,+ vi.fn(),+ );+ await flush();+ return res;+}++beforeEach(() => {+ vi.clearAllMocks();+});++describe.each([+ {+ name: "research paper search",+ router: () => createResearchRouter(),+ path: "/papers",+ query: { query: "retrieval augmented generation" },+ },+ {+ name: "developer search",+ router: () => createDeveloperRouter(),+ path: "/search",+ query: { query: "retry with backoff" },+ },+])("US-only processing on $name", testCase => {+ it("asks for US-only processing and relays a refusal as the US-only error", async () => {+ mocks.fetchResearchUpstream.mockResolvedValue(+ upstream(403, { success: false, code: "FORBIDDEN", error: REFUSAL }),+ );++ const res = await callRoute(+ testCase.router(),+ testCase.path,+ testCase.query,+ { usOnly: true },+ );++ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(+ expect.objectContaining({+ headers: expect.objectContaining({ "firecrawl-us-only": "true" }),+ }),+ );+ expect(res.setHeader).toHaveBeenLastCalledWith(+ "content-type",+ "application/json",+ );+ expect(res.status).toHaveBeenCalledWith(403);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: REFUSAL,+ });+ });++ it("passes other upstream errors through unchanged", async () => {+ const body = { success: false, code: "NOT_FOUND", error: "No match." };+ mocks.fetchResearchUpstream.mockResolvedValue(upstream(404, body));++ const res = await callRoute(+ testCase.router(),+ testCase.path,+ testCase.query,+ { usOnly: true },+ );++ expect(res.status).toHaveBeenCalledWith(404);+ expect(res.json).toHaveBeenCalledWith(body);+ });++ it("leaves a team without the restriction unchanged", async () => {+ const body = { success: false, code: "FORBIDDEN", error: "Forbidden." };+ mocks.fetchResearchUpstream.mockResolvedValue(upstream(403, body));++ const res = await callRoute(+ testCase.router(),+ testCase.path,+ testCase.query,+ {},+ );++ expect(+ mocks.fetchResearchUpstream.mock.calls[0][0].headers,+ ).not.toHaveProperty("firecrawl-us-only");+ expect(res.status).toHaveBeenCalledWith(403);+ expect(res.json).toHaveBeenCalledWith(body);+ });+});diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 6c1033474..f6afd65c7 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -7,6 +7,7 @@ import {fetchGovUpstream,fetchResearchUpstream,usOnlySkippedWarning,+ usOnlyUpstreamRefusal,} from "../../lib/research-upstream";import { isUsOnly } from "../../lib/us-only";import { chargeKeylessCredits } from "../../lib/keyless";@@ -446,6 +447,20 @@ function createResearchController(if (value) res.setHeader(h, value);}+ if (isUsOnly(authedReq.acuc?.flags) && upstream.status === 403) {+ const refusal = await usOnlyUpstreamRefusal(+ upstream,+ "This endpoint is not available for teams restricted to US-only processing.",+ );+ error = refusal.message;+ res.setHeader("content-type", "application/json");+ return res.status(403).json({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: refusal.message,+ });+ }+const text = await upstream.text();try {responseBody = text ? JSON.parse(text) : null;diff --git a/apps/api/src/routes/labs.ts b/apps/api/src/routes/labs.tsindex aa62f6746..1c7eb8d9e 100644--- a/apps/api/src/routes/labs.ts+++ b/apps/api/src/routes/labs.ts@@ -3,6 +3,7 @@ import { Readable } from "node:stream";import { Agent, fetch } from "undici";import { config } from "../config";import { logger as rootLogger } from "../lib/logger";+import { isUsOnly } from "../lib/us-only";import type { RequestWithAuth } from "../controllers/v1/types";import { RateLimiterMode } from "../types";import { authMiddleware, wrap } from "./shared";@@ -95,6 +96,16 @@ async function labsProxyController(req: Request, res: Response) {return labsError(res, 403, "This endpoint is not enabled for this team.");}+ // Every labs route is served from a hosted vector index, so none has a US-only route.+ if (isUsOnly(authedReq.acuc?.flags)) {+ return res.status(403).json({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error:+ "Labs Search is not available for teams restricted to US-only processing.",+ });+ }+try {const upstream = await fetch(base + req.originalUrl, {method: req.method,