Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): route US-only teams' search, research, exchange and agent requests (#5546)
Gergő Móricz57ac35524d9f
diff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.tsindex 21eb815a8..4d7bdbf68 100644--- a/apps/api/src/__tests__/snips/v2/us-only.test.ts+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts@@ -1,5 +1,13 @@import { describeIf, TEST_PRODUCTION } from "../lib";-import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";+import {+ scrape,+ scrapeRaw,+ scrapeTimeout,+ search,+ searchRaw,+ idmux,+ Identity,+} from "./lib";describeIf(TEST_PRODUCTION)("US-only teams", () => {let identity: Identity;@@ -77,3 +85,72 @@ describeIf(TEST_PRODUCTION)("US-only teams", () => {scrapeTimeout,);});++describeIf(TEST_PRODUCTION)("US-only teams: search", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "us-only/search",+ concurrency: 100,+ credits: 1000000,+ flags: { usOnly: true },+ });+ }, 10000 + scrapeTimeout);++ it.concurrent(+ "returns web results",+ async () => {+ const response = await search({ query: "firecrawl", limit: 5 }, identity);++ expect(response.web).toBeDefined();+ expect(response.web!.length).toBeGreaterThan(0);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "drops image results with a warning",+ async () => {+ const response = await searchRaw(+ {+ query: "firecrawl",+ limit: 5,+ sources: ["web", "images"],+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.data.web.length).toBeGreaterThan(0);+ expect(response.body.data.images ?? []).toHaveLength(0);+ expect(response.body.warning).toContain(+ "Image search is not available for teams restricted to US-only processing.",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses an image-only search",+ async () => {+ const response = await searchRaw(+ {+ query: "firecrawl",+ limit: 5,+ sources: ["images"],+ },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");+ expect(response.body.error).toContain(+ "Image search is not available for teams restricted to US-only processing.",+ );+ },+ scrapeTimeout,+ );+});diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex ada26d479..e8c44d6ee 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -32,6 +32,8 @@ import {SAFE_MODE_V0_UNSUPPORTED_MESSAGE,} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { isUsOnly } from "../../lib/us-only";+import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";async function searchHelper(jobId: string,@@ -76,6 +78,7 @@ async function searchHelper(lang: searchOptions.lang ?? "en",country: searchOptions.country ?? "us",location: searchOptions.location,+ usOnly: isUsOnly(flags),});let justSearch = pageOptions.fetchPageContent === false;@@ -327,6 +330,9 @@ export async function searchController(req: Request, res: Response) {if (error instanceof ScrapeJobTimeoutError) {return res.status(408).json({ error: error.message });}+ if (error instanceof UsOnlyUnavailableError) {+ return res.status(403).json({ code: error.code, error: error.message });+ }logger.error("Unhandled error occurred in search", { error });return res.status(500).json({ error: error.message });diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 486ced3af..cf2bd030b 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -18,6 +18,7 @@ import {removeDuplicateUrls,} from "../../lib/validateUrl";import { fireEngineMap } from "../../search/fireEngine";+import { isUsOnly } from "../../lib/us-only";import { billTeam } from "../../services/billing/credit_billing";import { logMap, logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";@@ -175,7 +176,8 @@ export async function getMapResults({Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,);- const cacheKey = `fireEngineMap:${mapUrl}`;+ const usOnly = isUsOnly(flags);+ const cacheKey = `fireEngineMap:${usOnly ? "us-only:" : ""}${mapUrl}`;const cachedResult = ignoreCache ? null : await redis.get(cacheKey);let allResults: any[] = [];@@ -193,6 +195,7 @@ export async function getMapResults({{numResults: resultsPerPage,page: page,+ usOnly,},abort,);diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 57bc740e1..4487c922c 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -46,6 +46,8 @@ import {} from "../../lib/keyless";import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { isUsOnly } from "../../lib/us-only";+import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";// Used for deep researchexport async function searchAndScrapeSearchResult(@@ -67,6 +69,7 @@ export async function searchAndScrapeSearchResult(query,logger,num_results: 5,+ usOnly: isUsOnly(flags),});const { scrapeOptions } = fromV1ScrapeOptions(@@ -428,6 +431,14 @@ export async function searchController(});}+ if (error instanceof UsOnlyUnavailableError) {+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }+logger.error("Unhandled error occurred in search", {version: "v1",error,diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 1672bcbf1..e0150c5e4 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -27,6 +27,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { fetchAgentThread, threadErrorFor } from "./agent-thread";+import { isUsOnly } from "../../lib/us-only";import {AGENT_DEFAULT_MAX_CREDITS,gateAgentCredits,@@ -329,6 +330,7 @@ export async function agentController(...(zeroDataRetentionMinutes !== null? { zeroDataRetentionMinutes }: {}),+ ...(isUsOnly(req.acuc?.flags) ? { usOnly: true } : {}),}),},);diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex c1552f307..51e9c197d 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -6,7 +6,9 @@ import { logger as rootLogger } from "../../lib/logger";import {fetchGovUpstream,fetchResearchUpstream,+ usOnlySkippedWarning,} from "../../lib/research-upstream";+import { isUsOnly } from "../../lib/us-only";import { chargeKeylessCredits } from "../../lib/keyless";import {indexDailyLimitRefusal,@@ -332,6 +334,7 @@ async function fetchForRequest(}headers["firecrawl-team-id"] = req.auth.team_id;headers["firecrawl-zero-data-retention"] = String(zeroDataRetention);+ if (isUsOnly(req.acuc?.flags)) headers["firecrawl-us-only"] = "true";if ("fetchUpstream" in endpoint) {return endpoint.fetchUpstream(params, headers);@@ -501,6 +504,16 @@ function createResearchController(options.legacy && upstream.ok? addLegacySnakeCaseAliases(responseBody): responseBody;+ const usOnlyWarning =+ upstream.ok && isUsOnly(authedReq.acuc?.flags)+ ? usOnlySkippedWarning(upstream.headers)+ : undefined;+ if (usOnlyWarning && !Array.isArray(response)) {+ response.warning =+ typeof response.warning === "string" && response.warning.length > 0+ ? `${response.warning} ${usOnlyWarning}`+ : usOnlyWarning;+ }return res.status(statusCode).json(response);} catch (err: unknown) {if (isResearchTimeoutError(err)) {diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex f12f2ae9b..801987624 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -1,6 +1,6 @@import { Response } from "express";import { providerScrapeController } from "./scrape-alexandria";-import { discoverTools } from "../../search/alexandria";+import { discoverTools, usOnlyToolsWarning } from "../../search/alexandria";import { config } from "../../config";import { timingSafeSecretEqual } from "../../lib/secret-compare";import { logger as _logger } from "../../lib/logger";@@ -54,6 +54,7 @@ import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { markAlexandriaActivity } from "../../lib/alexandria-activity";import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";+import { isUsOnly } from "../../lib/us-only";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -763,6 +764,7 @@ export async function scrapeController({teamId: req.auth.team_id,connectors: req.acuc?.flags?.exchangeRetrieve === true,+ usOnly: isUsOnly(req.acuc?.flags),toolDetail: req.body.toolDetail,urls: [...new Set(@@ -783,6 +785,14 @@ export async function scrapeController(}): undefined;if (tools) markAlexandriaActivity(req.auth.team_id);+ const toolsWarning = [+ tools?.warning,+ tools?.usOnlyExcluded+ ? usOnlyToolsWarning(tools.usOnlyExcluded)+ : undefined,+ ]+ .filter(Boolean)+ .join(" ");return res.status(200).json({success: true,@@ -797,7 +807,7 @@ export async function scrapeController(},...(tools ? { tools: tools.items } : {}),},- ...(tools?.warning ? { warning: tools.warning } : {}),+ ...(toolsWarning ? { warning: toolsWarning } : {}),scrape_id: origin?.includes("website") ? jobId : undefined,});},diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 6f43f2d41..d7adaa691 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -66,6 +66,12 @@ import {autoToolsEnabled,queryInferenceEnabled,} from "../../search/tool-intent";+import { isUsOnly } from "../../lib/us-only";+import { UsOnlyUpstreamRefusalError } from "../../lib/research-upstream";+import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";++const IMAGE_SEARCH_US_ONLY_MESSAGE =+ "Image search is not available for teams restricted to US-only processing.";const RESEARCH_CATEGORY_NOTICE: Notice = {message:@@ -251,6 +257,27 @@ async function searchControllerInner(});}+ // Image search has no US-only route: it is dropped with a warning, or+ // refused when nothing else was asked for.+ let usOnlyImagesWarning: string | undefined;+ const sources = req.body.sources as Array<{ type: string }>;+ if (+ isUsOnly(req.acuc?.flags) &&+ sources.some(source => source.type === "images")+ ) {+ req.body.sources = sources.filter(+ source => source.type !== "images",+ ) as SearchRequest["sources"];+ if (req.body.sources.length === 0 && !indexCategory) {+ return res.status(403).json({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: IMAGE_SEARCH_US_ONLY_MESSAGE,+ });+ }+ usOnlyImagesWarning = IMAGE_SEARCH_US_ONLY_MESSAGE;+ }+// Safe Mode: validate the per-request param up front and reject scrape// options it forbids (the worker backstop would otherwise strip them// silently). Domain controls force threat protection over the results.@@ -629,12 +656,19 @@ async function searchControllerInner(result.response.tools,);}+ const warning = [+ usOnlyImagesWarning,+ result.usOnlyWarning,+ result.toolsWarning,+ ]+ .filter(Boolean)+ .join(" ");return res.status(200).json({success: true,data: result.response,creditsUsed: result.totalCredits,id: jobId,- ...(result.toolsWarning ? { warning: result.toolsWarning } : {}),+ ...(warning ? { warning } : {}),});} catch (error) {if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {@@ -661,6 +695,26 @@ async function searchControllerInner(});}+ if (error instanceof UsOnlyUnavailableError) {+ setSpanAttributes(span, { "search.status_code": 403 });+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }++ if (error instanceof UsOnlyUpstreamRefusalError) {+ setSpanAttributes(span, { "search.status_code": error.status });+ return res.status(error.status).json({+ success: false,+ ...(error.status === 403+ ? { code: "US_ONLY_UNAVAILABLE" as const }+ : {}),+ error: error.message,+ });+ }+logger.error("Unhandled error occurred in search", {version: "v2",error,diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex ce73d6265..255434bb6 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -41,6 +41,7 @@ import { SourceTracker_F0 } from "./helpers/source-tracker-f0";import { getACUCTeam } from "../../../controllers/auth";import { orgIdFromAcuc } from "../../team-org";import { resolveThreatProtection } from "../../threat-protection/request";+import { isUsOnly } from "../../us-only";interface ExtractServiceOptions {request: ExtractRequest;@@ -128,6 +129,7 @@ export async function performExtraction_F0(query: rephrasedPrompt.replace('"', "").replace("'", ""),logger,num_results: 10,+ usOnly: isUsOnly(acuc?.flags),});request.urls = searchResults.map(result => result.url) as string[];diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 9727c1b86..5afe506c7 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -16,6 +16,7 @@ import {resolveRedirects,} from "./validateUrl";import { fireEngineMap } from "../search/fireEngine";+import { isUsOnly } from "./us-only";import { redisEvictConnection } from "../services/redis";import {generateURLSplits,@@ -218,7 +219,8 @@ export async function getMapResults({Math.min(maxFireEngineResults, limit) / resultsPerPage,);- const cacheKey = `fireEngineMap:${mapUrl}`;+ const usOnly = isUsOnly(flags);+ const cacheKey = `fireEngineMap:${usOnly ? "us-only:" : ""}${mapUrl}`;const cachedResult = ignoreCache? null: await redisEvictConnection.get(cacheKey);@@ -229,6 +231,7 @@ export async function getMapResults({{numResults: resultsPerPage,page,+ usOnly,},abort,);diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsindex cbb094224..0cd0cfd27 100644--- a/apps/api/src/lib/research-upstream.ts+++ b/apps/api/src/lib/research-upstream.ts@@ -1,4 +1,4 @@-import { Agent, fetch } from "undici";+import { Agent, fetch, type Response } from "undici";import { config } from "../config";const TIMEOUT_MS = 120_000;@@ -48,6 +48,40 @@ export async function fetchResearchUpstream(options: {});}+/** An upstream's refusal of a US-only request, relayed to the caller with its status. */+export class UsOnlyUpstreamRefusalError extends Error {+ constructor(+ readonly status: number,+ message: string,+ ) {+ super(message);+ }+}++export async function usOnlyUpstreamRefusal(+ upstream: Response,+ fallback: string,+): Promise<UsOnlyUpstreamRefusalError> {+ const body: any = await upstream.json().catch(() => null);+ const message = [body?.detail, body?.error, body?.message, body?.title].find(+ value => typeof value === "string" && value.length > 0,+ );+ return new UsOnlyUpstreamRefusalError(upstream.status, message ?? fallback);+}++/** The warning for the parts an upstream skipped for a US-only request, if it skipped any. */+export function usOnlySkippedWarning(headers: {+ get(name: string): string | null;+}): string | undefined {+ const skipped = (headers.get("firecrawl-us-only-skipped") ?? "")+ .split(",")+ .map(part => part.trim())+ .filter(part => /^[\w-]+$/.test(part));+ return skipped.length > 0+ ? `Some results were not available under US-only processing: ${skipped.join(", ")}.`+ : undefined;+}+export async function fetchGovUpstream(options: {query: string;k?: number;diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.tsindex 2c89a1832..5482b05d5 100644--- a/apps/api/src/routes/exchange.ts+++ b/apps/api/src/routes/exchange.ts@@ -16,6 +16,7 @@ import { logger as rootLogger } from "../lib/logger";import type { RequestWithAuth } from "../controllers/v1/types";import { RateLimiterMode } from "../types";import { authMiddleware, wrap } from "./shared";+import { isUsOnly } from "../lib/us-only";const DISCOVER_TIMEOUT_MS = 10_000;const RETRIEVE_TIMEOUT_MS = 50_000;@@ -181,6 +182,9 @@ function exchangeProxy(...(authedReq.acuc?.flags?.exchangeRetrieve? { "x-exchange-connectors": "true" }: {}),+ ...(isUsOnly(authedReq.acuc?.flags)+ ? { "x-exchange-us-only": "true" }+ : {}),},body: hasBody ? JSON.stringify(req.body ?? {}) : undefined,signal: AbortSignal.timeout(timeout),@@ -229,6 +233,9 @@ async function recommendationsController(req: Request, res: Response) {...forwardedRequestHeaders(req),"content-type": "application/json","x-exchange-team-id": teamId,+ ...(isUsOnly(authedReq.acuc?.flags)+ ? { "x-exchange-us-only": "true" }+ : {}),},body: JSON.stringify(signals),signal: AbortSignal.timeout(DISCOVER_TIMEOUT_MS),diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.tsindex 7d6638a16..c6e66c68c 100644--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts@@ -13,6 +13,7 @@ import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";import { robustFetch } from "../lib/fetch";import { safeMarkdownToHtml } from "./pdf/markdownToHtml";import { EngineError, ExchangeRefusedError } from "../error";+import { isUsOnly } from "../../../lib/us-only";// The enrichment waterfall's record of which provider matched and every// provider it tried, in order.@@ -195,6 +196,9 @@ export async function scrapeURLWithExchange(const response = await robustFetch({url: `${config.FIRE_EXCHANGE_URL!.replace(/\/+$/, "")}/v1/scrape`,method: "POST",+ ...(isUsOnly(meta.internalOptions.teamFlags ?? undefined)+ ? { headers: { "x-exchange-us-only": "true" } }+ : {}),body: {requestId: meta.id,teamId: meta.internalOptions.teamId,diff --git a/apps/api/src/search/alexandria.test.ts b/apps/api/src/search/alexandria.test.tsindex caf989cfb..ababcfe90 100644--- a/apps/api/src/search/alexandria.test.ts+++ b/apps/api/src/search/alexandria.test.ts@@ -257,3 +257,29 @@ it("keeps the results of queries that answered", async () => {expect(result.items).toHaveLength(1);expect(result.warning).toBeUndefined();});++it("reports the tools the Exchange hid from a US-only team", async () => {+ request.mockResolvedValue({+ status: 200,+ body: {+ success: true,+ creditsCost: 0,+ data: { items: [tool], usOnlyExcluded: 3 },+ },+ });+ const result = await discoverTools({ ...input, usOnly: true }, logger);+ expect(request.mock.calls[0][0]).toMatchObject({ usOnly: true });+ expect(result.items).toHaveLength(1);+ expect(result.usOnlyExcluded).toBe(3);+ expect(result).not.toHaveProperty("warning");+});++it("reports no hidden tools when the Exchange hid nothing", async () => {+ request.mockResolvedValue({+ status: 200,+ body: { success: true, creditsCost: 0, data: { items: [tool] } },+ });+ const result = await discoverTools({ ...input, usOnly: true }, logger);+ expect(result.items).toHaveLength(1);+ expect(result).not.toHaveProperty("usOnlyExcluded");+});diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsindex 87300097a..960cba262 100644--- a/apps/api/src/search/alexandria.ts+++ b/apps/api/src/search/alexandria.ts@@ -13,6 +13,8 @@ export const isAlexandriaSource = (source: { type: string }) =>type ToolDiscovery = {items: DiscoveredTool[];warning?: string;+ /** Tools the Exchange hid from a US-only team; set only when it hid some. */+ usOnlyExcluded?: number;/** Every discovery leg failed and nothing was found. */unavailable?: true;};@@ -40,6 +42,10 @@ export function isToolsOnlySearch(return true;}+export function usOnlyToolsWarning(excluded: number): string {+ return `${excluded} ${excluded === 1 ? "tool is" : "tools are"} not available for teams restricted to US-only processing.`;+}+export async function discoverTools(input: {teamId: string;@@ -52,6 +58,8 @@ export async function discoverTools(timeoutMs: number;/** The team may use its own MCP connectors: their listed tools join its discovery. */connectors?: boolean;+ /** The team may only be served by US-based providers. */+ usOnly?: boolean;},logger: Logger,): Promise<ToolDiscovery> {@@ -60,6 +68,8 @@ export async function discoverTools(const selected = { semantic: new Set<string>(), domain: new Set<string>() };let attempted = 0;let answered = 0;+ // Lookups can hide the same tools, so the largest count is reported.+ let usOnlyExcluded = 0;const remaining = (deadline: number) => {if (Date.now() >= deadline) throw new Error("Tool discovery timed out");return deadline - Date.now();@@ -71,6 +81,7 @@ export async function discoverTools(timeoutMs: remaining(deadline),maximumCredits: 0,connectors: input.connectors === true,+ usOnly: input.usOnly === true,body: {provider: "firecrawl",capability: "find-tools",@@ -87,7 +98,7 @@ export async function discoverTools(});if (result.status !== 200)throw new Error("Tool contract lookup unavailable");- return z+ const data = z.object({success: z.literal(true),creditsCost: z.literal(0),@@ -95,9 +106,17 @@ export async function discoverTools(items: z.array(input.toolDetail === "full" ? toolSchema : toolSummarySchema,),+ usOnlyExcluded: z+ .number()+ .int()+ .nonnegative()+ .optional()+ .catch(undefined),}),})- .parse(result.body).data.items;+ .parse(result.body).data;+ usOnlyExcluded = Math.max(usOnlyExcluded, data.usOnlyExcluded ?? 0);+ return data.items;};const merge = (tool: z.infer<typeof toolSchema> | z.infer<typeof toolSummarySchema>,@@ -191,6 +210,7 @@ export async function discoverTools(body: { urls: batch },timeoutMs: remaining(deadline),connectors: input.connectors === true,+ usOnly: input.usOnly === true,});if (result.status !== 200)throw new Error("Domain discovery unavailable");@@ -280,6 +300,7 @@ export async function discoverTools(...(answered < attempted? { warning: "Some tool discovery results are unavailable." }: {}),+ ...(input.usOnly && usOnlyExcluded > 0 ? { usOnlyExcluded } : {}),...(attempted && !answered && !items.size? { unavailable: true as const }: {}),diff --git a/apps/api/src/search/developer.test.ts b/apps/api/src/search/developer.test.tsindex 766ee979b..30c1785ff 100644--- a/apps/api/src/search/developer.test.ts+++ b/apps/api/src/search/developer.test.ts@@ -207,3 +207,58 @@ describe("searchDeveloperCategory", () => {).resolves.toEqual([]);});});++describe("searchDeveloperCategory for a US-only team", () => {+ const usOnlyOptions = {+ query: "retries",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ usOnly: true,+ };++ it("asks for US-only processing and reports the parts it skipped", async () => {+ fetchMock.mockResolvedValue({+ ...upstreamOk({ results: [{ url: "https://github.com/a/b" }] }),+ headers: new Headers({ "firecrawl-us-only-skipped": "web, docs" }),+ });+ const warnings: string[] = [];++ const results = await searchDeveloperCategory(+ { ...usOnlyOptions, warnings },+ logger,+ );++ expect(firstCall()[1].headers).toEqual({+ "firecrawl-team-id": "t1",+ "firecrawl-zero-data-retention": "false",+ "firecrawl-us-only": "true",+ });+ expect(results.map(result => result.url)).toEqual([+ "https://github.com/a/b",+ ]);+ expect(warnings).toEqual([+ "Some results were not available under US-only processing: web, docs.",+ ]);+ });++ it("relays an upstream refusal instead of returning no results", async () => {+ fetchMock.mockResolvedValue({+ ok: false,+ status: 403,+ json: async () => ({+ error:+ "Code search is not available for teams restricted to US-only processing.",+ }),+ } as any);++ await expect(+ searchDeveloperCategory(usOnlyOptions, logger),+ ).rejects.toMatchObject({+ status: 403,+ message:+ "Code search is not available for teams restricted to US-only processing.",+ });+ });+});diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.tsindex 91da7067d..bffa53b94 100644--- a/apps/api/src/search/developer.ts+++ b/apps/api/src/search/developer.ts@@ -1,6 +1,11 @@import type { Logger } from "winston";import type { WebSearchResult } from "../lib/entities";-import { fetchResearchUpstream } from "../lib/research-upstream";+import {+ fetchResearchUpstream,+ usOnlySkippedWarning,+ usOnlyUpstreamRefusal,+ UsOnlyUpstreamRefusalError,+} from "../lib/research-upstream";import { hasCategory, type CategoryOption } from "../lib/search-query-builder";const DEVELOPER_QUERY_KEYS = ["query", "k"];@@ -16,6 +21,9 @@ export async function searchDeveloperCategory(teamId: string;timeout: number;zeroDataRetention: boolean;+ usOnly?: boolean;+ /** Receives the warnings for parts the upstream skipped for a US-only request. */+ warnings?: string[];},logger: Logger,): Promise<WebSearchResult[]> {@@ -27,12 +35,19 @@ export async function searchDeveloperCategory(headers: {"firecrawl-team-id": options.teamId,"firecrawl-zero-data-retention": String(options.zeroDataRetention),+ ...(options.usOnly ? { "firecrawl-us-only": "true" } : {}),},timeoutMs: options.timeout,});if (upstream === null) {return [];}+ if (options.usOnly && upstream.status >= 400 && upstream.status < 500) {+ throw await usOnlyUpstreamRefusal(+ upstream,+ "Developer search refused the request.",+ );+ }if (!upstream.ok) {await upstream.body?.cancel().catch(() => {});logger.warn("Developer category upstream failed", {@@ -41,6 +56,11 @@ export async function searchDeveloperCategory(return [];}+ const skipped = options.usOnly+ ? usOnlySkippedWarning(upstream.headers)+ : undefined;+ if (skipped) options.warnings?.push(skipped);+const body: any = await upstream.json();const results: any[] = Array.isArray(body?.results) ? body.results : [];// Exact WebSearchResult shape: developer results are indistinguishable@@ -65,6 +85,7 @@ export async function searchDeveloperCategory(})).filter(result => result.url.length > 0);} catch (error) {+ if (error instanceof UsOnlyUpstreamRefusalError) throw error;logger.warn("Developer category upstream error", { error });return [];}diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex b3d9dd007..dc1f531e2 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -17,6 +17,7 @@ vi.mock("./tool-intent", () => ({vi.mock("./alexandria", () => ({discoverTools: mocks.discoverTools,+ usOnlyToolsWarning: (excluded: number) => `${excluded} tools hidden`,isAlexandriaSource: (source: { type: string }) =>source.type === "alexandria",}));@@ -247,6 +248,32 @@ describe("executeSearch tool intent", () => {expect(result.searchCredits).toBe(2);});+ it("warns a US-only team about tools hidden from automatic discovery", async () => {+ mocks.search.mockResolvedValue({ web: [webResult] });+ mocks.inferToolIntent.mockResolvedValue({+ toolQueries: ["US unemployment rate statistics"],+ domains: [],+ needsTools: true,+ });+ mocks.discoverTools.mockResolvedValue({+ items: [{ id: "fred/series/observations" }],+ usOnlyExcluded: 2,+ });+ const result = await executeSearch(+ { ...options([]), autoTools: true },+ { ...context, flags: { usOnly: true } },+ logger,+ );+ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ usOnly: true }),+ );+ expect(mocks.discoverTools).toHaveBeenCalledWith(+ expect.objectContaining({ usOnly: true }),+ logger,+ );+ expect(result.toolsWarning).toBe("2 tools hidden");+ });+it("leaves a default search alone when the query does not need tools", async () => {mocks.search.mockResolvedValue({ web: [webResult] });mocks.inferToolIntent.mockResolvedValue({diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 40548fd3e..e9fb7fe9c 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -1,5 +1,9 @@import type { Logger } from "winston";-import { discoverTools, isAlexandriaSource } from "./alexandria";+import {+ discoverTools,+ isAlexandriaSource,+ usOnlyToolsWarning,+} from "./alexandria";import {AUTO_TOOLS_WAIT_MS,inferToolIntent,@@ -46,6 +50,7 @@ import { checkUrlsAgainstThreatPolicy } from "../lib/threat-protection/request";import { calculateThreatScanCredits } from "../lib/scrape-billing";import { config } from "../config";import { logger as rootLogger } from "../lib/logger";+import { isUsOnly } from "../lib/us-only";interface SearchOptions {query: string;@@ -94,6 +99,8 @@ interface SearchContext {}interface SearchExecuteResult {+ /** Parts of the search a US-only team could not be served. */+ usOnlyWarning?: string;toolsWarning?: string;/** The tools were added automatically, not requested. */toolsAuto?: boolean;@@ -128,6 +135,8 @@ export async function executeSearch(} = context;const num_results_buffer = Math.floor(limit * 2);+ const usOnly = isUsOnly(flags);+ const usOnlyWarnings: string[] = [];logger.info("Searching for results");@@ -151,6 +160,7 @@ export async function executeSearch(toolDetail: options.toolDetail ?? "compact",teamId,connectors: flags?.exchangeRetrieve === true,+ usOnly,limit,queries: wantsTools? intent.toolQueries@@ -163,7 +173,10 @@ export async function executeSearch(},logger,);- return verifyTools(query, found.items, logger);+ return {+ items: await verifyTools(query, found.items, logger),+ usOnlyExcluded: found.usOnlyExcluded ?? 0,+ };}).catch(() => null): null;@@ -175,6 +188,7 @@ export async function executeSearch(toolDetail: options.toolDetail ?? "compact",teamId,connectors: flags?.exchangeRetrieve === true,+ usOnly,limit,queries: wantsTools ? [query] : undefined,urls:@@ -226,6 +240,7 @@ export async function executeSearch(options.enterprise?.some(mode => mode === "zdr" || mode === "anon",) === true,+ ...(usOnly ? { usOnly, warnings: usOnlyWarnings } : {}),},logger,)@@ -257,6 +272,7 @@ export async function executeSearch(...(options.excludeDomains ?? []),...operators.excludedSites.filter(site => !site.includes("/")),]),+ usOnly,})) as SearchV2Response);const keepOperatorMatches = <T extends { url?: string; title?: string; position?: number },@@ -419,7 +435,7 @@ export async function executeSearch(tool,]),);- for (const tool of extra ?? []) {+ for (const tool of extra?.items ?? []) {const key = `${tool.provider}/${tool.capability}`;const previous = merged.get(key);merged.set(@@ -444,14 +460,28 @@ export async function executeSearch();}searchResponse.tools = [...merged.values()];- toolsWarning = discovery.warning;- toolsUnavailable = discovery.unavailable === true && !extra?.length;+ // Both legs can hide the same tools, so the larger count is reported.+ const usOnlyExcluded = Math.max(+ discovery.usOnlyExcluded ?? 0,+ extra?.usOnlyExcluded ?? 0,+ );+ toolsWarning =+ [+ discovery.warning,+ usOnlyExcluded > 0 ? usOnlyToolsWarning(usOnlyExcluded) : undefined,+ ]+ .filter(Boolean)+ .join(" ") || undefined;+ toolsUnavailable = discovery.unavailable === true && !extra?.items.length;} else if (extraDiscovery) {const extra = await waitForExtra();- if (extra?.length) {- searchResponse.tools = extra;+ if (extra?.items.length) {+ searchResponse.tools = extra.items;toolsAuto = true;}+ if (extra?.usOnlyExcluded) {+ toolsWarning = usOnlyToolsWarning(extra.usOnlyExcluded);+ }}const isZDR = options.enterprise?.includes("zdr");@@ -611,6 +641,9 @@ export async function executeSearch(return {response: searchResponse,+ ...(usOnlyWarnings.length > 0+ ? { usOnlyWarning: usOnlyWarnings.join(" ") }+ : {}),toolsWarning,toolsUnavailable,toolsAuto,diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex 6070285f5..7effc28cb 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -18,6 +18,7 @@ export async function fire_engine_search(location?: string;numResults: number;page?: number;+ usOnly?: boolean;},abort?: AbortSignal,): Promise<SearchResult[]> {@@ -33,6 +34,7 @@ export async function fire_engine_search(tbs: options.tbs,numResults: options.numResults,page: options.page ?? 1,+ ...(options.usOnly ? { usOnly: true } : {}),};const url = `${config.FIRE_ENGINE_BETA_URL}/search`;@@ -57,6 +59,7 @@ export async function fireEngineMap(location?: string;numResults: number;page?: number;+ usOnly?: boolean;},abort?: AbortSignal,): Promise<SearchResult[]> {@@ -75,6 +78,7 @@ export async function fireEngineMap(tbs: options.tbs,numResults: options.numResults,page: options.page ?? 1,+ ...(options.usOnly ? { usOnly: true } : {}),};const url = `${config.FIRE_ENGINE_BETA_URL}/map`;diff --git a/apps/api/src/search/gov.ts b/apps/api/src/search/gov.tsindex e6cce255c..2456aa835 100644--- a/apps/api/src/search/gov.ts+++ b/apps/api/src/search/gov.ts@@ -1,6 +1,11 @@import type { Logger } from "winston";import type { WebSearchResult } from "../lib/entities";-import { fetchGovUpstream } from "../lib/research-upstream";+import {+ fetchGovUpstream,+ usOnlySkippedWarning,+ usOnlyUpstreamRefusal,+ UsOnlyUpstreamRefusalError,+} from "../lib/research-upstream";import { hasCategory, type CategoryOption } from "../lib/search-query-builder";export function wantsGovCategory(categories?: CategoryOption[]): boolean {@@ -14,6 +19,9 @@ export async function searchGovCategory(teamId: string;timeout: number;zeroDataRetention: boolean;+ usOnly?: boolean;+ /** Receives the warnings for parts the upstream skipped for a US-only request. */+ warnings?: string[];},logger: Logger,): Promise<WebSearchResult[]> {@@ -24,18 +32,30 @@ export async function searchGovCategory(headers: {"firecrawl-team-id": options.teamId,"firecrawl-zero-data-retention": String(options.zeroDataRetention),+ ...(options.usOnly ? { "firecrawl-us-only": "true" } : {}),},timeoutMs: options.timeout,});if (upstream === null) {return [];}+ if (options.usOnly && upstream.status >= 400 && upstream.status < 500) {+ throw await usOnlyUpstreamRefusal(+ upstream,+ "Government search refused the request.",+ );+ }if (!upstream.ok) {await upstream.body?.cancel().catch(() => {});logger.warn("Gov category upstream failed", { status: upstream.status });return [];}+ const skipped = options.usOnly+ ? usOnlySkippedWarning(upstream.headers)+ : undefined;+ if (skipped) options.warnings?.push(skipped);+const body: any = await upstream.json();const results: any[] = Array.isArray(body?.data?.web) ? body.data.web : [];return results@@ -55,6 +75,7 @@ export async function searchGovCategory(})).filter(result => result.url.length > 0);} catch (error) {+ if (error instanceof UsOnlyUpstreamRefusalError) throw error;logger.warn("Gov category upstream error", { error });return [];}diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.tsindex 027724329..45bf8af7f 100644--- a/apps/api/src/search/index.ts+++ b/apps/api/src/search/index.ts@@ -4,6 +4,7 @@ import { searxng_search } from "./searxng";import { fire_engine_search } from "./fireEngine";import { Logger } from "winston";import { ddgSearch } from "./v2/ddgsearch";+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";export async function search({query,@@ -18,6 +19,7 @@ export async function search({proxy = undefined,sleep_interval = 0,timeout = 5000,+ usOnly = false,}: {query: string;logger: Logger;@@ -31,7 +33,14 @@ export async function search({proxy?: string;sleep_interval?: number;timeout?: number;+ usOnly?: boolean;}): Promise<SearchResult[]> {+ // The fallback backends below take no US-only routing.+ if (usOnly && !config.FIRE_ENGINE_BETA_URL) {+ throw new UsOnlyUnavailableError(+ "Search is not available for teams restricted to US-only processing on this deployment.",+ );+ }try {if (config.FIRE_ENGINE_BETA_URL) {logger.info("Using fire engine search");@@ -42,6 +51,7 @@ export async function search({lang,country,location,+ usOnly,});return results;}diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 8441df7c7..95c88d60a 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -49,6 +49,7 @@ export async function fire_engine_search_v2(enterprise?: ("default" | "anon" | "zdr")[];includeDomains?: string[];excludeDomains?: string[];+ usOnly?: boolean;},abort?: AbortSignal,): Promise<SearchV2Response> {@@ -72,6 +73,7 @@ export async function fire_engine_search_v2(safe: options.safe ? ("active" as const) : undefined,includeDomains: options.includeDomains,excludeDomains: options.excludeDomains,+ ...(options.usOnly ? { usOnly: true } : {}),};const requestedTypes = normalizeSearchTypes(options.type);diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex aa60b170d..d8ef982cf 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -4,6 +4,7 @@ import { fire_engine_search_v2 } from "./fireEngine-v2";import { searxng_search } from "./searxng";import { ddgSearch } from "./ddgsearch";import { Logger } from "winston";+import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";export async function search({query,@@ -24,6 +25,7 @@ export async function search({enterprise = undefined,includeDomains = undefined,excludeDomains = undefined,+ usOnly = false,}: {query: string;logger: Logger;@@ -43,7 +45,14 @@ export async function search({enterprise?: ("default" | "anon" | "zdr")[];includeDomains?: string[];excludeDomains?: string[];+ usOnly?: boolean;}): Promise<SearchV2Response> {+ // The fallback backends below take no US-only routing.+ if (usOnly && !config.FIRE_ENGINE_BETA_URL) {+ throw new UsOnlyUnavailableError(+ "Search is not available for teams restricted to US-only processing on this deployment.",+ );+ }try {if (config.FIRE_ENGINE_BETA_URL) {logger.info("Using fire engine search");@@ -60,6 +69,7 @@ export async function search({enterprise,includeDomains,excludeDomains,+ usOnly,});return results;diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.tsindex 91b9d6513..becd24e19 100644--- a/apps/api/src/services/alexandria/client.ts+++ b/apps/api/src/services/alexandria/client.ts@@ -27,6 +27,8 @@ export async function exchangeRequest(input: {*/connectors?: boolean;zeroDataRetention?: boolean;+ /** The team may only be served by US-based providers. */+ usOnly?: boolean;}): Promise<ExchangeResponse> {if (!config.FIRE_EXCHANGE_URL)throw new Error("Alexandria is not configured");@@ -57,6 +59,7 @@ export async function exchangeRequest(input: {...(input.zeroDataRetention? { "x-exchange-zero-data-retention": "true" }: {}),+ ...(input.usOnly ? { "x-exchange-us-only": "true" } : {}),...(input.requestId ? { "x-request-id": input.requestId } : {}),...(input.maximumCredits === undefined? {}diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 7cca9e577..00f434a2b 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -23,6 +23,7 @@ import { getBillingQueue } from "../queue-service";import { redisRateLimitClient } from "../rate-limiter";import { authorizeProviders } from "./access";import { exchangeRequest } from "./client";+import { isUsOnly } from "../../lib/us-only";import {answerSchema,refusal,@@ -186,12 +187,14 @@ export async function retrieveProviders(input: {const id = hash([input.teamId, input.requestId]);const key = `alexandria:retrieve:${id}`;const zeroDataRetention = input.zeroDataRetention === true;- // Appended only when set so existing non-ZDR fingerprints stay stable.+ const usOnly = isUsOnly(input.flags);+ // Appended only when set so existing fingerprints stay stable.const fingerprint = hash([...(termsOnly? [input.calls, billable, termsIdentity ?? null]: [input.calls, billable]),...(zeroDataRetention ? [{ zeroDataRetention }] : []),+ ...(usOnly ? [{ usOnly }] : []),]);const record: Retrieval = {fingerprint,@@ -352,6 +355,7 @@ export async function retrieveProviders(input: {billingCountry: input.billingCountry,connectors: input.flags?.exchangeRetrieve === true,...(zeroDataRetention ? { zeroDataRetention } : {}),+ ...(usOnly ? { usOnly } : {}),}).catch(error => {logger.warn("Provider quote unavailable", { error, chargeId: id });return undefined;@@ -479,6 +483,7 @@ export async function retrieveProviders(input: {: {}),...(termsIdentity ? { termsIdentity } : {}),...(zeroDataRetention ? { zeroDataRetention } : {}),+ ...(usOnly ? { usOnly } : {}),maximumCredits,}).catch(error => {throw new Error(`Alexandria did not answer: ${error?.message ?? error}`);diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 9f18e9e4a..0de4b34a2 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -76,6 +76,7 @@ import { monitorTelemetryMetadata } from "./search/tuning";import { computeGoalVersion } from "./search/dedupe";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { getACUCTeam } from "../../controllers/auth";+import { isUsOnly } from "../../lib/us-only";import type { TeamFlags } from "../../controllers/v1/types";import { orgIdForTeam } from "../../lib/team-org";import {@@ -887,6 +888,7 @@ async function runMonitorSearchTarget(params: {knownPages,knownEvents,zeroDataRetention: false,+ usOnly: isUsOnly(teamFlags),logger: logger.child({monitorId: monitor.id,checkId: check.id,diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex c21406ea2..00322583c 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -278,6 +278,7 @@ export async function runSearchTarget(params: {knownPages: Map<string, KnownPage>;knownEvents: KnownEvent[];zeroDataRetention: boolean;+ usOnly?: boolean;logger: Logger;}): Promise<SearchTargetRunResult> {const { target, knownPages, goalVersion, logger } = params;@@ -324,6 +325,7 @@ export async function runSearchTarget(params: {logger,num_results: target.maxResults,tbs,+ usOnly: params.usOnly,},logger,);