Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): refuse labs search for US-only teams and relay research refusals as US_ONLY_UNAVAILABLE (#5557)

Gergő Móricz973f7c3535cb
diff --git a/apps/api/src/__tests__/routes/labs.routes.test.ts b/apps/api/src/__tests__/routes/labs.routes.test.ts
index d682a3242..761ab9039 100644
--- a/apps/api/src/__tests__/routes/labs.routes.test.ts
+++ b/apps/api/src/__tests__/routes/labs.routes.test.ts
@@ -19,8 +19,22 @@ vi.mock("../../lib/logger", () => {
return { logger };
});
+const mocks = vi.hoisted(() => ({
+ fetch: vi.fn(),
+ flags: {} as Record<string, unknown>,
+}));
+
+vi.mock("undici", async importOriginal => ({
+ ...(await importOriginal<typeof import("undici")>()),
+ fetch: mocks.fetch,
+}));
+
vi.mock("../../routes/shared", () => ({
- authMiddleware: () => (_req: any, _res: any, next: any) => next(),
+ authMiddleware: () => (req: any, _res: any, next: any) => {
+ req.auth = { team_id: "team-1" };
+ req.acuc = { flags: mocks.flags };
+ next();
+ },
wrap: (fn: any) => fn,
}));
@@ -96,3 +110,50 @@ describe("labs router", () => {
expect(response.body.code).not.toBe("LABS_ROUTE_NOT_PROXIED");
});
});
+
+describe("labs router for a US-only team", () => {
+ beforeEach(() => {
+ mocks.fetch.mockReset();
+ });
+
+ it.each(LABS_ROUTES)(
+ "refuses %s without calling the labs service",
+ async route => {
+ mocks.flags = { labsSearch: true, usOnly: true };
+ const [method, path] = route.split(" ");
+
+ const response = await (request(appWithLabs()) as any)[
+ method.toLowerCase()
+ ]("/labs" + path.replace(/:\w+/g, "id-1"));
+
+ expect(response.status).toBe(403);
+ expect(response.body).toEqual({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error:
+ "Labs Search is not available for teams restricted to US-only processing.",
+ });
+ expect(mocks.fetch).not.toHaveBeenCalled();
+ },
+ );
+
+ it("still proxies for a team without the US-only restriction", async () => {
+ mocks.flags = { labsSearch: true };
+ mocks.fetch.mockResolvedValue(
+ new Response(JSON.stringify({ success: true, results: [] }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+
+ const response = await request(appWithLabs()).post("/labs/search");
+
+ expect(response.status).toBe(200);
+ expect(response.body).toEqual({ success: true, results: [] });
+ expect(mocks.fetch).toHaveBeenCalledTimes(1);
+ expect(mocks.fetch.mock.calls[0][0]).toBe("https://labs.test/labs/search");
+ expect(mocks.fetch.mock.calls[0][1].headers).toMatchObject({
+ "x-labs-search-team-id": "team-1",
+ });
+ });
+});
diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-us-only.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-us-only.test.ts
new file mode 100644
index 000000000..bacffa169
--- /dev/null
+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-us-only.test.ts
@@ -0,0 +1,172 @@
+import { vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ fetchResearchUpstream: vi.fn(),
+}));
+
+vi.mock("../../../services/logging/log_job", () => ({
+ logRequest: vi.fn().mockResolvedValue(undefined),
+ logResearchEndpoint: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../lib/research-upstream", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../../lib/research-upstream")>()),
+ fetchResearchUpstream: mocks.fetchResearchUpstream,
+}));
+
+vi.mock("../../../services/billing/credit_billing", () => ({
+ billTeam: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../lib/keyless", () => ({
+ chargeKeylessCredits: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../lib/index-daily-limit", () => ({
+ reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),
+ indexDailyLimitRefusal: vi.fn(),
+}));
+
+vi.mock("../../../lib/logger", () => ({
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+}));
+
+import { createDeveloperRouter, createResearchRouter } from "../research-proxy";
+
+const REFUSAL =
+ "Paper search is not available for teams restricted to US-only processing.";
+const flush = () => new Promise(resolve => setImmediate(resolve));
+
+function upstream(status: number, body: unknown) {
+ return {
+ ok: status >= 200 && status < 300,
+ status,
+ headers: new Headers({ "content-type": "application/json" }),
+ text: async () => JSON.stringify(body),
+ json: async () => body,
+ };
+}
+
+async function callRoute(
+ router: any,
+ path: string,
+ query: Record<string, unknown>,
+ flags: Record<string, unknown>,
+) {
+ const layer = router.stack.find(
+ (candidate: any) =>
+ candidate.route?.path === path && candidate.route?.methods?.get,
+ );
+ const res: any = {
+ status: vi.fn(),
+ json: vi.fn(),
+ send: vi.fn(),
+ end: vi.fn(),
+ setHeader: vi.fn(),
+ };
+ res.status.mockReturnValue(res);
+ res.json.mockReturnValue(res);
+ res.send.mockReturnValue(res);
+ layer.route.stack.at(-1).handle(
+ {
+ method: "GET",
+ query,
+ params: {},
+ body: {},
+ headers: {},
+ auth: { team_id: "11111111-1111-1111-1111-111111111111" },
+ acuc: { api_key_id: 7, flags },
+ },
+ res,
+ vi.fn(),
+ );
+ await flush();
+ return res;
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+describe.each([
+ {
+ name: "research paper search",
+ router: () => createResearchRouter(),
+ path: "/papers",
+ query: { query: "retrieval augmented generation" },
+ },
+ {
+ name: "developer search",
+ router: () => createDeveloperRouter(),
+ path: "/search",
+ query: { query: "retry with backoff" },
+ },
+])("US-only processing on $name", testCase => {
+ it("asks for US-only processing and relays a refusal as the US-only error", async () => {
+ mocks.fetchResearchUpstream.mockResolvedValue(
+ upstream(403, { success: false, code: "FORBIDDEN", error: REFUSAL }),
+ );
+
+ const res = await callRoute(
+ testCase.router(),
+ testCase.path,
+ testCase.query,
+ { usOnly: true },
+ );
+
+ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headers: expect.objectContaining({ "firecrawl-us-only": "true" }),
+ }),
+ );
+ expect(res.setHeader).toHaveBeenLastCalledWith(
+ "content-type",
+ "application/json",
+ );
+ expect(res.status).toHaveBeenCalledWith(403);
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error: REFUSAL,
+ });
+ });
+
+ it("passes other upstream errors through unchanged", async () => {
+ const body = { success: false, code: "NOT_FOUND", error: "No match." };
+ mocks.fetchResearchUpstream.mockResolvedValue(upstream(404, body));
+
+ const res = await callRoute(
+ testCase.router(),
+ testCase.path,
+ testCase.query,
+ { usOnly: true },
+ );
+
+ expect(res.status).toHaveBeenCalledWith(404);
+ expect(res.json).toHaveBeenCalledWith(body);
+ });
+
+ it("leaves a team without the restriction unchanged", async () => {
+ const body = { success: false, code: "FORBIDDEN", error: "Forbidden." };
+ mocks.fetchResearchUpstream.mockResolvedValue(upstream(403, body));
+
+ const res = await callRoute(
+ testCase.router(),
+ testCase.path,
+ testCase.query,
+ {},
+ );
+
+ expect(
+ mocks.fetchResearchUpstream.mock.calls[0][0].headers,
+ ).not.toHaveProperty("firecrawl-us-only");
+ expect(res.status).toHaveBeenCalledWith(403);
+ expect(res.json).toHaveBeenCalledWith(body);
+ });
+});
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 6c1033474..f6afd65c7 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -7,6 +7,7 @@ import {
fetchGovUpstream,
fetchResearchUpstream,
usOnlySkippedWarning,
+ usOnlyUpstreamRefusal,
} from "../../lib/research-upstream";
import { isUsOnly } from "../../lib/us-only";
import { chargeKeylessCredits } from "../../lib/keyless";
@@ -446,6 +447,20 @@ function createResearchController(
if (value) res.setHeader(h, value);
}
+ if (isUsOnly(authedReq.acuc?.flags) && upstream.status === 403) {
+ const refusal = await usOnlyUpstreamRefusal(
+ upstream,
+ "This endpoint is not available for teams restricted to US-only processing.",
+ );
+ error = refusal.message;
+ res.setHeader("content-type", "application/json");
+ return res.status(403).json({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error: refusal.message,
+ });
+ }
+
const text = await upstream.text();
try {
responseBody = text ? JSON.parse(text) : null;
diff --git a/apps/api/src/routes/labs.ts b/apps/api/src/routes/labs.ts
index aa62f6746..1c7eb8d9e 100644
--- a/apps/api/src/routes/labs.ts
+++ b/apps/api/src/routes/labs.ts
@@ -3,6 +3,7 @@ import { Readable } from "node:stream";
import { Agent, fetch } from "undici";
import { config } from "../config";
import { logger as rootLogger } from "../lib/logger";
+import { isUsOnly } from "../lib/us-only";
import type { RequestWithAuth } from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
import { authMiddleware, wrap } from "./shared";
@@ -95,6 +96,16 @@ async function labsProxyController(req: Request, res: Response) {
return labsError(res, 403, "This endpoint is not enabled for this team.");
}
+ // Every labs route is served from a hosted vector index, so none has a US-only route.
+ if (isUsOnly(authedReq.acuc?.flags)) {
+ return res.status(403).json({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error:
+ "Labs Search is not available for teams restricted to US-only processing.",
+ });
+ }
+
try {
const upstream = await fetch(base + req.originalUrl, {
method: req.method,