Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): pin US-only teams' page fetches to the US (#5594)

Gergő Móriczf9729cf88ac2
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 132b13901..102de5b1f 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -381,6 +381,17 @@ jobs:
--reporter=default
working-directory: apps/api
+ - name: Run US-only page fetch regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/lib/us-only.test.ts
+ src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts
+ src/scraper/scrapeURL/__tests__/usOnly.test.ts
+ src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run country check route regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=default
diff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.ts
index 9e8b19aee..88c80ed34 100644
--- a/apps/api/src/__tests__/snips/v2/us-only.test.ts
+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts
@@ -9,6 +9,8 @@ import {
Identity,
crawlStart,
extractRaw,
+ map,
+ browserCreateRaw,
} from "./lib";
import { config } from "../../../config";
@@ -200,6 +202,86 @@ describeIf(TEST_PRODUCTION)("US-only teams", () => {
scrapeTimeout,
);
+ it.concurrent(
+ "scrapes with a US location",
+ async () => {
+ const response = await scrape(
+ { url: "https://firecrawl.dev", location: { country: "US" } },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses a scrape located outside the US",
+ async () => {
+ const response = await scrapeRaw(
+ {
+ url: "https://firecrawl.dev",
+ location: { country: "NL" },
+ timeout: scrapeTimeout,
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.success).toBe(false);
+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");
+ expect(response.body.error).toContain(
+ 'location.country "NL" isn\'t available to teams restricted to US-only processing',
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses a crawl located outside the US",
+ async () => {
+ const response = await crawlStart(
+ {
+ url: "https://firecrawl.dev",
+ scrapeOptions: { location: { country: "NL" } },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses a map located outside the US",
+ async () => {
+ const response = await map(
+ { url: "https://firecrawl.dev", location: { country: "NL" } },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses a browser session located outside the US",
+ async () => {
+ const response = await browserCreateRaw(
+ { location: { country: "NL" } },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");
+ },
+ scrapeTimeout,
+ );
+
it.concurrent(
"refuses X/Twitter URLs",
async () => {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 2188943b3..3117651b8 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -49,6 +49,7 @@ import {
} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { requestCreditsShards } from "../../lib/request-credits-store";
+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
export async function crawlController(req: Request, res: Response) {
try {
@@ -216,6 +217,17 @@ export async function crawlController(req: Request, res: Response) {
});
}
+ const usOnlyLocationError = usOnlyLocationRefusal(
+ auth.chunk?.flags ?? undefined,
+ pageOptions?.geolocation?.country,
+ );
+ if (usOnlyLocationError) {
+ return res.status(403).json({
+ code: "US_ONLY_UNAVAILABLE",
+ error: usOnlyLocationError,
+ });
+ }
+
// if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?
// try {
// const a = new WebScraperDataProvider();
@@ -253,6 +265,9 @@ export async function crawlController(req: Request, res: Response) {
);
internalOptions.disableSmartWaitCache = true; // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
internalOptions.orgId = auth.chunk?.org_id ?? null;
+ if (isUsOnly(auth.chunk?.flags ?? undefined)) {
+ internalOptions.teamFlags = auth.chunk?.flags;
+ }
internalOptions.saveScrapeResultToGCS = process.env
.GCS_FIRE_ENGINE_BUCKET_NAME
? true
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 257da3ddb..e8941e40a 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -39,6 +39,7 @@ import {
SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
async function scrapeHelper(
jobId: string,
@@ -52,6 +53,7 @@ async function scrapeHelper(
): Promise<{
success: boolean;
error?: string;
+ code?: string;
data?: V0Document | { url: string };
returnCode: number;
}> {
@@ -88,6 +90,19 @@ async function scrapeHelper(
};
}
+ const usOnlyLocationError = usOnlyLocationRefusal(
+ acuc?.flags ?? undefined,
+ pageOptions?.geolocation?.country,
+ );
+ if (usOnlyLocationError) {
+ return {
+ success: false,
+ error: usOnlyLocationError,
+ code: "US_ONLY_UNAVAILABLE",
+ returnCode: 403,
+ };
+ }
+
const { scrapeOptions, internalOptions } = fromV0Combo(
pageOptions,
extractorOptions,
@@ -97,6 +112,9 @@ async function scrapeHelper(
);
internalOptions.orgId = acuc?.org_id ?? null;
+ if (isUsOnly(acuc?.flags ?? undefined)) {
+ internalOptions.teamFlags = acuc?.flags;
+ }
internalOptions.saveScrapeResultToGCS = process.env
.GCS_FIRE_ENGINE_BUCKET_NAME
? true
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index e8c44d6ee..042038c7c 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -91,6 +91,9 @@ async function searchHelper(
team_id,
);
internalOptions.orgId = org_id;
+ if (isUsOnly(flags)) {
+ internalOptions.teamFlags = flags;
+ }
if (justSearch) {
const searchCredits = Math.ceil(res.length / 10) * 2;
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 4487c922c..750dfda77 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -46,7 +46,7 @@ import {
} from "../../lib/keyless";
import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
-import { isUsOnly } from "../../lib/us-only";
+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
// Used for deep research
@@ -200,6 +200,22 @@ export async function searchController(
});
}
+ const usOnlyLocationError =
+ requestedFormats.length > 0
+ ? usOnlyLocationRefusal(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.location?.country,
+ req.body.scrapeOptions?.geolocation?.country,
+ )
+ : undefined;
+ if (usOnlyLocationError) {
+ return res.status(403).json({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error: usOnlyLocationError,
+ });
+ }
+
// Search only scrapes (and only honors scrapeOptions) when formats are
// requested, so the scrape-option checks apply only then.
if (
diff --git a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
index f7474925a..9a1b2b2c2 100644
--- a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
@@ -772,6 +772,84 @@ describe("scrapeInteractController", () => {
expect(res.status).toHaveBeenCalledWith(status);
expect(getHangarRecording).not.toHaveBeenCalled();
});
+
+ describe("US-only teams", () => {
+ const usOnlyRequest = (body: Record<string, unknown>) =>
+ ({
+ body,
+ headers: {},
+ auth: { team_id: "team-123" },
+ acuc: { flags: { usOnly: true } },
+ }) as any;
+
+ beforeEach(() => {
+ vi.mocked(createHangarBrowser).mockResolvedValue({
+ id: "br_us",
+ max_expires_at: 700,
+ cdp_url: "wss://hangar.example/cdp",
+ } as any);
+ vi.mocked(insertBrowserSession).mockImplementation(
+ async row => row as any,
+ );
+ });
+
+ it("refuses a browser session located outside the US", async () => {
+ const res = buildRes();
+ await browserCreateController(
+ usOnlyRequest({ location: { country: "NL" } }),
+ res,
+ );
+ expect(res.status).toHaveBeenCalledWith(403);
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error: expect.stringContaining('location.country "NL"'),
+ });
+ expect(createHangarBrowser).not.toHaveBeenCalled();
+ });
+
+ it("asks Hangar for a US-only exit", async () => {
+ const res = buildRes();
+ await browserCreateController(
+ usOnlyRequest({ location: { country: "us" } }),
+ res,
+ );
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ success: true }),
+ );
+ expect(createHangarBrowser).toHaveBeenCalledWith(
+ expect.any(String),
+ "team-123",
+ expect.objectContaining({ usOnly: true }),
+ );
+ });
+
+ it("refuses to interact from a scrape located outside the US", async () => {
+ vi.mocked(readScrapeJobState).mockResolvedValueOnce({
+ status: "completed",
+ requestId: "scrape-123",
+ completedAtMs: Date.now(),
+ creditsBilled: 1,
+ replay: { targetUrl: "https://example.com", waitForMs: 0, actions: [] },
+ browser: { blockAds: true, location: { country: "nl" } },
+ } as any);
+ const res = buildRes();
+ await scrapeInteractController(
+ {
+ ...usOnlyRequest({ code: "console.log('ok')" }),
+ params: { jobId: "scrape-123" },
+ },
+ res,
+ );
+ expect(res.status).toHaveBeenCalledWith(403);
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error: expect.stringContaining('location.country "NL"'),
+ });
+ expect(createHangarBrowser).not.toHaveBeenCalled();
+ });
+ });
});
vi.mock("../../../services/redlock", () => ({ redlock: { using: vi.fn() } }));
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index 7c1219aeb..cbc13b2f7 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -36,6 +36,7 @@ import {
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
import { browserProfileNameSchema } from "../../lib/browser-profiles";
import { withZeroDataRetention } from "../../lib/otel-tracer";
+import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
export const browserCreateRequestSchema = z.object({
ttl: z.number().int().min(30).max(3600).default(600),
@@ -94,6 +95,13 @@ export function browserError(
signup_url: prompt.signup_url,
});
}
+ if (error instanceof UsOnlyUnavailableError) {
+ return res.status(403).json({
+ success: false,
+ code: error.code,
+ error: error.message,
+ });
+ }
const knownError =
error instanceof BrowserSessionError || error instanceof HangarError;
return res.status(knownError ? error.status : 502).json({
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 06465c149..7ca2d8686 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -521,6 +521,17 @@ async function createSessionForScrape(
});
return { session };
} catch (error) {
+ if (error instanceof UsOnlyUnavailableError) {
+ return {
+ error: true as const,
+ status: 403,
+ body: {
+ success: false as const,
+ code: error.code,
+ error: error.message,
+ },
+ };
+ }
logger.error("Failed to initialize scrape browser session", { error });
const knownError =
error instanceof BrowserSessionError || error instanceof HangarError;
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 06ba0b523..aeb519786 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -70,7 +70,7 @@ import {
autoToolsEnabled,
queryInferenceEnabled,
} from "../../search/tool-intent";
-import { isUsOnly } from "../../lib/us-only";
+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
import { UsOnlyUpstreamRefusalError } from "../../lib/research-upstream";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
@@ -337,6 +337,21 @@ async function searchControllerInner(
});
}
+ const usOnlyLocationError =
+ requestedFormats.length > 0
+ ? usOnlyLocationRefusal(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.location?.country,
+ )
+ : undefined;
+ if (usOnlyLocationError) {
+ return res.status(403).json({
+ success: false,
+ code: "US_ONLY_UNAVAILABLE",
+ error: usOnlyLocationError,
+ });
+ }
+
// Search only scrapes (and only honors scrapeOptions) when formats are
// requested, so the scrape-option checks apply only then.
if (
diff --git a/apps/api/src/lib/browser-lifecycle.ts b/apps/api/src/lib/browser-lifecycle.ts
index f4746bc9a..de2e035f1 100644
--- a/apps/api/src/lib/browser-lifecycle.ts
+++ b/apps/api/src/lib/browser-lifecycle.ts
@@ -55,6 +55,8 @@ import { getScrapeZDR } from "./zdr-helpers";
import { withZeroDataRetention } from "./otel-tracer";
import { redlock } from "../services/redlock";
import { redisRateLimitClient } from "../services/rate-limiter";
+import { isUsOnly, usOnlyLocationRefusal } from "./us-only";
+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";
const logger = rootLogger;
@@ -157,6 +159,12 @@ export async function createBrowserSession(
initialize?: (browserId: string) => Promise<void>;
},
) {
+ const usOnlyLocationError = usOnlyLocationRefusal(
+ req.acuc?.flags,
+ options.location?.country,
+ );
+ if (usOnlyLocationError)
+ throw new UsOnlyUnavailableError(usOnlyLocationError);
const zeroDataRetention =
getBrowserZDR(req) || options.zeroDataRetention === true;
if (zeroDataRetention && (options.recordSession || options.profile)) {
@@ -242,6 +250,7 @@ async function createBrowserSessionInternal(
const browser = await createHangarBrowser(id, req.auth.team_id, {
...options,
zeroDataRetention,
+ usOnly: isUsOnly(req.acuc?.flags),
});
browserId = browser.id;
if (options.initialize) await options.initialize(browser.id);
diff --git a/apps/api/src/lib/browser-options.test.ts b/apps/api/src/lib/browser-options.test.ts
index 900b56620..369670900 100644
--- a/apps/api/src/lib/browser-options.test.ts
+++ b/apps/api/src/lib/browser-options.test.ts
@@ -13,6 +13,7 @@ import { createHangarBrowser } from "./hangar";
async function hangarCreateBody(
options: BrowserOptions,
zeroDataRetention = false,
+ usOnly = false,
) {
const fetchMock = vi.fn(async (_url: string, _init: RequestInit) =>
Response.json({
@@ -29,6 +30,7 @@ async function hangarCreateBody(
streamWebView: true,
recordSession: false,
zeroDataRetention,
+ usOnly,
});
return JSON.parse(fetchMock.mock.calls[0][1].body as string);
}
@@ -137,6 +139,39 @@ describe("Hangar create body under zero data retention", () => {
});
});
+describe("Hangar create body for US-only teams", () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("always asks for a US exit", async () => {
+ expect(
+ (await hangarCreateBody({ blockAds: true }, false, true)).proxy,
+ ).toEqual({
+ country: "us",
+ type: "basic",
+ });
+ expect(
+ (
+ await hangarCreateBody(
+ browserOptionsFromScrape({ proxy: "stealth" }),
+ false,
+ true,
+ )
+ ).proxy,
+ ).toEqual({ country: "us", type: "mobile" });
+ });
+
+ it("keeps ZDR exits within the US", async () => {
+ const body = await hangarCreateBody({ blockAds: true }, true, true);
+ expect(body.proxy).toEqual({
+ country: "us",
+ type: "basic",
+ zero_data_retention: true,
+ });
+ });
+});
+
describe("isBrowserOptions", () => {
it("accepts options persisted before location and proxy were stored", () => {
expect(isBrowserOptions({ blockAds: true })).toBe(true);
diff --git a/apps/api/src/lib/browser-options.ts b/apps/api/src/lib/browser-options.ts
index d36b33e97..96e9ddf24 100644
--- a/apps/api/src/lib/browser-options.ts
+++ b/apps/api/src/lib/browser-options.ts
@@ -55,6 +55,7 @@ export function isBrowserOptions(value: unknown): value is BrowserOptions {
export function hangarBrowserOptions(
options: BrowserOptions,
zeroDataRetention: boolean,
+ usOnly = false,
) {
return {
// uBlock Origin Lite covers ads, trackers and cookie notices.
@@ -69,11 +70,19 @@ export function hangarBrowserOptions(
: {}),
// Scrape's stealth proxy is Hangar's mobile pool. Omitted fields keep
// Hangar's defaults: country "us", type "basic". Under ZDR, Hangar picks
- // only ZDR exits and fails the browser if none is available.
- ...(options.location || options.proxy === "stealth" || zeroDataRetention
+ // only ZDR exits and fails the browser if none is available. A US-only
+ // team always asks for a US exit.
+ ...(options.location ||
+ options.proxy === "stealth" ||
+ zeroDataRetention ||
+ usOnly
? {
proxy: {
- ...(options.location ? { country: options.location.country } : {}),
+ ...(usOnly
+ ? { country: "us" }
+ : options.location
+ ? { country: options.location.country }
+ : {}),
type: options.proxy === "stealth" ? "mobile" : "basic",
...(zeroDataRetention ? { zero_data_retention: true } : {}),
},
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 9bf1becd1..c8b3ca7ee 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -9,6 +9,7 @@ import type { Logger } from "winston";
import { withSpan, setSpanAttributes } from "./otel-tracer";
import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";
import { resolveSafeMode } from "./safe-mode";
+import { isUsOnly } from "./us-only";
import {
firstPipelineError,
reportPipelineError,
@@ -1060,6 +1061,7 @@ export function crawlToCrawler(
currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,
zeroDataRetention:
(getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,
+ usOnly: isUsOnly(sc.internalOptions?.teamFlags) || isUsOnly(teamFlags),
location: sc.scrapeOptions?.location,
headers: sc.scrapeOptions?.headers,
robotsUserAgent: sc.crawlerOptions?.robotsUserAgent,
diff --git a/apps/api/src/lib/hangar.ts b/apps/api/src/lib/hangar.ts
index 0550ab570..580bf3660 100644
--- a/apps/api/src/lib/hangar.ts
+++ b/apps/api/src/lib/hangar.ts
@@ -113,6 +113,7 @@ export async function createHangarBrowser(
streamWebView: boolean;
recordSession: boolean;
zeroDataRetention: boolean;
+ usOnly?: boolean;
},
): Promise<HangarCreated> {
const body = {
@@ -125,7 +126,7 @@ export async function createHangarBrowser(
},
recording: { enabled: options.recordSession },
execution: { enabled: true },
- ...hangarBrowserOptions(options, options.zeroDataRetention),
+ ...hangarBrowserOptions(options, options.zeroDataRetention, options.usOnly),
};
let created: HangarCreated | undefined;
for (let attempt = 0; attempt < 3; attempt++) {
diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.ts
index 69e72efec..6eb1afd52 100644
--- a/apps/api/src/lib/permissions.ts
+++ b/apps/api/src/lib/permissions.ts
@@ -16,6 +16,7 @@ import {
THREAT_PROTECTION_NOT_ENABLED_MESSAGE,
THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,
} from "./threat-protection/request";
+import { usOnlyLocationRefusal } from "./us-only";
type LocationOptions = { country?: string };
type ThreatProtectionOption = { mode?: string };
@@ -23,8 +24,10 @@ type ThreatProtectionOption = { mode?: string };
interface APIRequest {
zeroDataRetention?: boolean;
location?: LocationOptions;
+ geolocation?: LocationOptions;
scrapeOptions?: {
location?: LocationOptions;
+ geolocation?: LocationOptions;
threatProtection?: ThreatProtectionOption;
};
crawlerOptions?: {
@@ -171,6 +174,17 @@ export function checkPermissions(
}
}
+ const usOnlyLocationError = usOnlyLocationRefusal(
+ flags,
+ request.location?.country,
+ request.geolocation?.country,
+ request.scrapeOptions?.location?.country,
+ request.scrapeOptions?.geolocation?.country,
+ );
+ if (usOnlyLocationError) {
+ return { error: usOnlyLocationError, code: "US_ONLY_UNAVAILABLE" };
+ }
+
// ip whitelist perms
const needsWhitelist =
request.location?.country === "us-whitelist" ||
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index d27d6d59d..2fb037c4f 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -19,6 +19,7 @@ export async function fetchRobotsTxt(
{
url,
zeroDataRetention,
+ usOnly,
location,
headers,
skipCache,
@@ -26,6 +27,7 @@ export async function fetchRobotsTxt(
}: {
url: string;
zeroDataRetention: boolean;
+ usOnly?: boolean;
skipTlsVerification?: boolean;
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
@@ -90,6 +92,7 @@ export async function fetchRobotsTxt(
: undefined,
teamId: "robots-txt",
zeroDataRetention,
+ ...(usOnly ? { teamFlags: { usOnly: true } } : {}),
},
new CostTracking(),
);
diff --git a/apps/api/src/lib/us-only.test.ts b/apps/api/src/lib/us-only.test.ts
new file mode 100644
index 000000000..9b7cd69ac
--- /dev/null
+++ b/apps/api/src/lib/us-only.test.ts
@@ -0,0 +1,82 @@
+import { checkPermissions } from "./permissions";
+import { usOnlyLocationRefusal } from "./us-only";
+
+const usOnly = { usOnly: true };
+
+describe("usOnlyLocationRefusal", () => {
+ it("refuses a country outside the US for a US-only team", () => {
+ expect(usOnlyLocationRefusal(usOnly, "nl")).toBe(
+ 'location.country "NL" isn\'t available to teams restricted to US-only processing, because pages are fetched only from US exits. Omit location or set it to "US".',
+ );
+ });
+
+ it.each(["us", "US", "us-generic", "US-generic", "", null, 42, undefined])(
+ "allows %s",
+ country => {
+ expect(usOnlyLocationRefusal(usOnly, country)).toBeUndefined();
+ },
+ );
+
+ it("refuses us-whitelist, which isn't the US", () => {
+ expect(usOnlyLocationRefusal(usOnly, "us-whitelist")).toContain(
+ 'location.country "US-WHITELIST"',
+ );
+ });
+
+ it("names the first country outside the US", () => {
+ expect(
+ usOnlyLocationRefusal(usOnly, "us", undefined, "fr", "de"),
+ ).toContain('"FR"');
+ });
+
+ it("leaves teams without the restriction alone", () => {
+ expect(usOnlyLocationRefusal(undefined, "nl")).toBeUndefined();
+ expect(usOnlyLocationRefusal(null, "nl")).toBeUndefined();
+ expect(usOnlyLocationRefusal({ usOnly: false }, "nl")).toBeUndefined();
+ });
+});
+
+describe("checkPermissions for US-only teams", () => {
+ it.each([
+ ["location", { location: { country: "nl" } }],
+ ["v1 geolocation", { geolocation: { country: "NL" } }],
+ [
+ "scrapeOptions.location",
+ { scrapeOptions: { location: { country: "nl" } } },
+ ],
+ [
+ "scrapeOptions.geolocation",
+ { scrapeOptions: { geolocation: { country: "NL" } } },
+ ],
+ ])("refuses a non-US %s with US_ONLY_UNAVAILABLE", (_, request) => {
+ const result = checkPermissions(request, usOnly);
+ expect(result.code).toBe("US_ONLY_UNAVAILABLE");
+ expect(result.error).toContain('location.country "NL"');
+ });
+
+ it("refuses us-whitelist with US_ONLY_UNAVAILABLE", () => {
+ expect(
+ checkPermissions({ location: { country: "us-whitelist" } }, usOnly).code,
+ ).toBe("US_ONLY_UNAVAILABLE");
+ });
+
+ it("allows a US location or none", () => {
+ expect(checkPermissions({ location: { country: "us" } }, usOnly)).toEqual(
+ {},
+ );
+ expect(
+ checkPermissions({ location: { country: "us-generic" } }, usOnly),
+ ).toEqual({});
+ expect(
+ checkPermissions({ geolocation: { country: "US-generic" } }, usOnly),
+ ).toEqual({});
+ expect(checkPermissions({}, usOnly)).toEqual({});
+ });
+
+ it("leaves other teams' locations alone", () => {
+ expect(checkPermissions({ location: { country: "nl" } }, null)).toEqual({});
+ expect(
+ checkPermissions({ location: { country: "nl" } }, { usOnly: false }),
+ ).toEqual({});
+ });
+});
diff --git a/apps/api/src/lib/us-only.ts b/apps/api/src/lib/us-only.ts
index 10f924639..8bd245ca7 100644
--- a/apps/api/src/lib/us-only.ts
+++ b/apps/api/src/lib/us-only.ts
@@ -5,6 +5,29 @@ export function isUsOnly(flags: TeamFlags | undefined): boolean {
return flags?.usOnly === true;
}
+const US_COUNTRIES = ["us", "us-generic"];
+
+/**
+ * The refusal for a US-only team asking for a country other than the US, since
+ * its pages are fetched only from US exits. Undefined for an unrestricted team
+ * or when every country given is "us" or "us-generic" (the schema default).
+ */
+export function usOnlyLocationRefusal(
+ flags: TeamFlags | undefined,
+ ...countries: unknown[]
+): string | undefined {
+ if (!isUsOnly(flags)) return undefined;
+ const country = countries.find(
+ (value): value is string =>
+ typeof value === "string" &&
+ value !== "" &&
+ !US_COUNTRIES.includes(value.toLowerCase()),
+ );
+ return country === undefined
+ ? undefined
+ : `location.country "${country.toUpperCase()}" isn't available to teams restricted to US-only processing, because pages are fetched only from US exits. Omit location or set it to "US".`;
+}
+
/** The `data_region` value that pins a US-only team's GPU work to US capacity. */
export const US_DATA_REGION = "us";
diff --git a/apps/api/src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts b/apps/api/src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts
new file mode 100644
index 000000000..c36e35d2d
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts
@@ -0,0 +1,128 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../../scrapeURL", () => ({
+ scrapeURL: vi.fn(async () => ({ success: false, error: "unreachable" })),
+}));
+
+import { scrapeURL } from "../../scrapeURL";
+import { crawlToCrawler } from "../../../lib/crawl-redis";
+import { getLinksFromSitemap } from "../sitemap";
+import { scrapeSitemap } from "../../crawler/sitemap";
+import { logger } from "../../../lib/logger";
+
+const storedCrawl = (teamFlags?: { usOnly?: boolean }) =>
+ ({
+ originUrl: "https://example.com/",
+ crawlerOptions: {},
+ scrapeOptions: {},
+ internalOptions: { teamId: "team-a", orgId: null, teamFlags },
+ team_id: "team-a",
+ createdAt: 0,
+ }) as any;
+
+const internalOptionsSent = () =>
+ vi
+ .mocked(scrapeURL)
+ .mock.calls.map(([, , , internalOptions]) => internalOptions);
+
+beforeEach(() => {
+ vi.mocked(scrapeURL).mockClear();
+});
+
+describe("crawl discovery for US-only teams", () => {
+ it("marks the crawler US-only from the stored crawl or the team's live flags", () => {
+ expect(
+ crawlToCrawler("crawl-1", storedCrawl({ usOnly: true }), null).usOnly,
+ ).toBe(true);
+ expect(
+ crawlToCrawler("crawl-1", storedCrawl(), { usOnly: true }).usOnly,
+ ).toBe(true);
+ expect(crawlToCrawler("crawl-1", storedCrawl(), null).usOnly).toBe(false);
+ });
+
+ it("fetches robots.txt as US-only", async () => {
+ await crawlToCrawler(
+ "crawl-1",
+ storedCrawl({ usOnly: true }),
+ null,
+ ).getRobotsTxt();
+ expect(internalOptionsSent()).toEqual([
+ expect.objectContaining({
+ teamId: "robots-txt",
+ teamFlags: { usOnly: true },
+ }),
+ ]);
+ });
+
+ it("fetches other teams' robots.txt without team flags", async () => {
+ await crawlToCrawler("crawl-1", storedCrawl(), null).getRobotsTxt();
+ expect(internalOptionsSent()).toHaveLength(1);
+ expect(internalOptionsSent()[0]).not.toHaveProperty("teamFlags");
+ });
+
+ it("fetches nested sitemaps as US-only", async () => {
+ vi.mocked(scrapeURL).mockResolvedValueOnce({
+ success: true,
+ document: {
+ rawHtml:
+ '<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>https://example.com/sitemap-posts.xml</loc></sitemap></sitemapindex>',
+ metadata: { statusCode: 200 },
+ },
+ } as any);
+ await getLinksFromSitemap(
+ {
+ sitemapUrl: "https://example.com/sitemap.xml",
+ urlsHandler: () => {},
+ mode: "fire-engine",
+ zeroDataRetention: false,
+ usOnly: true,
+ },
+ logger,
+ "crawl-1",
+ new Set(),
+ );
+ expect(vi.mocked(scrapeURL).mock.calls.map(([, url]) => url)).toEqual([
+ "https://example.com/sitemap.xml",
+ "https://example.com/sitemap-posts.xml",
+ ]);
+ expect(internalOptionsSent()).toEqual([
+ expect.objectContaining({ teamFlags: { usOnly: true } }),
+ expect.objectContaining({ teamFlags: { usOnly: true } }),
+ ]);
+ });
+
+ it("fetches sitemaps as US-only", async () => {
+ await getLinksFromSitemap(
+ {
+ sitemapUrl: "https://example.com/sitemap.xml",
+ urlsHandler: () => {},
+ mode: "fire-engine",
+ zeroDataRetention: false,
+ usOnly: true,
+ },
+ logger,
+ "crawl-1",
+ new Set(),
+ );
+ await expect(
+ scrapeSitemap({
+ url: "https://example.com/sitemap-2.xml",
+ maxAge: 0,
+ zeroDataRetention: false,
+ usOnly: true,
+ location: undefined,
+ crawlId: "crawl-1",
+ }),
+ ).rejects.toThrow();
+ expect(internalOptionsSent()).toEqual([
+ expect.objectContaining({
+ teamId: "sitemap",
+ teamFlags: { usOnly: true },
+ }),
+ expect.objectContaining({
+ teamId: "sitemap",
+ teamFlags: { usOnly: true },
+ }),
+ ]);
+ });
+});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 980f3fe85..fe51024ca 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -85,6 +85,7 @@ export class WebCrawler {
private maxDiscoveryDepth: number | undefined;
private currentDiscoveryDepth: number;
private zeroDataRetention: boolean;
+ public readonly usOnly: boolean;
private location?: ScrapeOptions["location"];
private headers?: Record<string, string>;
private robotsUserAgent?: string;
@@ -109,6 +110,7 @@ export class WebCrawler {
maxDiscoveryDepth,
currentDiscoveryDepth,
zeroDataRetention,
+ usOnly,
location,
headers,
robotsUserAgent,
@@ -131,6 +133,7 @@ export class WebCrawler {
maxDiscoveryDepth?: number;
currentDiscoveryDepth?: number;
zeroDataRetention?: boolean;
+ usOnly?: boolean;
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
robotsUserAgent?: string;
@@ -155,6 +158,7 @@ export class WebCrawler {
this.ignoreRobotsTxt = ignoreRobotsTxt ?? false;
this.regexOnFullURL = regexOnFullURL ?? false;
this.zeroDataRetention = zeroDataRetention ?? false;
+ this.usOnly = usOnly ?? false;
this.logger = _logger.child({
crawlId: this.jobId,
module: "WebCrawler",
@@ -500,6 +504,7 @@ export class WebCrawler {
url: this.robotsTxtUrl,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ usOnly: this.usOnly,
skipTlsVerification,
...(this.robotsUserAgent
? {
@@ -941,6 +946,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ usOnly: this.usOnly,
headers: this.headers,
},
this.logger,
@@ -1016,6 +1022,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ usOnly: this.usOnly,
headers: this.headers,
},
this.logger,
@@ -1060,6 +1067,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ usOnly: this.usOnly,
headers: this.headers,
},
this.logger,
@@ -1088,6 +1096,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ usOnly: this.usOnly,
headers: this.headers,
},
this.logger,
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 198ba4cdc..64b0be975 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -28,6 +28,7 @@ export async function getLinksFromSitemap(
mode = "axios",
maxAge = 0,
zeroDataRetention,
+ usOnly,
location,
headers,
}: {
@@ -36,6 +37,7 @@ export async function getLinksFromSitemap(
mode?: "axios" | "fire-engine";
maxAge?: number;
zeroDataRetention: boolean;
+ usOnly?: boolean;
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
},
@@ -126,6 +128,7 @@ export async function getLinksFromSitemap(
: undefined,
teamId: "sitemap",
zeroDataRetention,
+ ...(usOnly ? { teamFlags: { usOnly: true } } : {}),
},
new CostTracking(),
);
@@ -213,6 +216,7 @@ export async function getLinksFromSitemap(
urlsHandler,
mode,
zeroDataRetention,
+ usOnly,
location,
headers,
maxAge,
@@ -246,6 +250,7 @@ export async function getLinksFromSitemap(
urlsHandler,
mode,
zeroDataRetention,
+ usOnly,
location,
headers,
maxAge,
@@ -295,6 +300,7 @@ export async function getLinksFromSitemap(
urlsHandler,
mode,
zeroDataRetention,
+ usOnly,
location,
headers,
maxAge,
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index f96fac663..9e6b681b4 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -21,6 +21,7 @@ type SitemapScrapeOptions = {
url: string;
maxAge: number;
zeroDataRetention: boolean;
+ usOnly?: boolean;
location: ScrapeOptions["location"];
crawlId: string;
logger?: Logger;
@@ -101,6 +102,7 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {
teamId: "sitemap",
orgId: null,
zeroDataRetention: options.zeroDataRetention,
+ ...(options.usOnly ? { teamFlags: { usOnly: true } } : {}),
crawlId: options.crawlId,
isPreCrawl: options.isPreCrawl,
},
diff --git a/apps/api/src/scraper/scrapeURL/__tests__/usOnly.test.ts b/apps/api/src/scraper/scrapeURL/__tests__/usOnly.test.ts
new file mode 100644
index 000000000..ec5881e9b
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/__tests__/usOnly.test.ts
@@ -0,0 +1,46 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../engines", async importOriginal => ({
+ ...(await importOriginal<typeof import("../engines")>()),
+ buildFallbackList: vi.fn(),
+ scrapeURLWithEngine: vi.fn(),
+}));
+
+import { scrapeURL } from "..";
+import { buildFallbackList, scrapeURLWithEngine } from "../engines";
+import { UsOnlyUnavailableError } from "../error";
+import { scrapeOptions } from "../../../controllers/v2/types";
+import { CostTracking } from "../../../lib/cost-tracking";
+
+const usOnlyTeam = {
+ teamId: "team-a",
+ orgId: null,
+ teamFlags: { usOnly: true },
+};
+
+beforeEach(() => {
+ vi.mocked(buildFallbackList).mockReset();
+ vi.mocked(scrapeURLWithEngine).mockReset();
+});
+
+describe("scrapeURL for US-only teams", () => {
+ it("refuses a location outside the US before any engine runs", async () => {
+ const result = await scrapeURL(
+ "us-only-location",
+ "https://example.com",
+ scrapeOptions.parse({ location: { country: "NL" } }),
+ { ...usOnlyTeam },
+ new CostTracking(),
+ );
+
+ expect(result.success).toBe(false);
+ expect(!result.success && result.error).toBeInstanceOf(
+ UsOnlyUnavailableError,
+ );
+ expect(!result.success && result.error.message).toContain(
+ 'location.country "NL"',
+ );
+ expect(buildFallbackList).not.toHaveBeenCalled();
+ expect(scrapeURLWithEngine).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts
new file mode 100644
index 000000000..4229c87a8
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts
@@ -0,0 +1,148 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const fake = vi.hoisted(() => ({ calls: [] as any[] }));
+
+vi.mock("../../../lib/fetch", () => ({
+ robustFetch: vi.fn(async (params: any) => {
+ fake.calls.push(params);
+ return { error: "Scrape failed" };
+ }),
+ redactRequestBodyForLog: (body: unknown) => body,
+}));
+
+vi.mock("../../../../../lib/gcs-jobs", () => ({
+ getDocFromGCS: vi.fn(async () => null),
+}));
+
+import { EngineError } from "../../../error";
+import {
+ scrapeURLWithFireEngineChromeCDP,
+ scrapeURLWithFireEngineTLSClient,
+} from "../index";
+import type { ResolvedSafeMode } from "../../../../../lib/safe-mode";
+
+const logger = {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child() {
+ return this;
+ },
+} as any;
+
+const safeMode: ResolvedSafeMode = {
+ lockdown: false,
+ domainControls: false,
+ enforceRobots: false,
+ disableStealthProxy: false,
+ disableAuthentication: false,
+ disableSiteHandling: true,
+ exposeWebdriver: false,
+ useHeadlessUserAgent: false,
+ disablePlatformSelection: false,
+ disableCountrySelection: false,
+ disableAutomaticReferrer: true,
+};
+
+function meta({
+ teamFlags,
+ location,
+ safeMode,
+}: {
+ teamFlags?: { usOnly?: boolean };
+ location?: { country?: string; languages?: string[] };
+ safeMode?: ResolvedSafeMode;
+}) {
+ return {
+ id: "scrape-1",
+ url: "https://example.com",
+ options: {
+ formats: [{ type: "markdown" }],
+ skipTlsVerification: false,
+ proxy: "auto",
+ location,
+ },
+ internalOptions: {
+ teamId: "team-a",
+ zeroDataRetention: false,
+ teamFlags,
+ safeMode,
+ },
+ logger,
+ abort: {
+ scrapeTimeout: () => 30000,
+ asSignal: () => undefined,
+ throwIfAborted: () => {},
+ },
+ featureFlags: new Set(),
+ mock: null,
+ } as any;
+}
+
+beforeEach(() => {
+ fake.calls = [];
+});
+
+describe.each([
+ ["chrome-cdp", scrapeURLWithFireEngineChromeCDP],
+ ["tlsclient", scrapeURLWithFireEngineTLSClient],
+] as const)("fire-engine %s requests", (_, scrapeWith) => {
+ const sentRequest = async (m: any) => {
+ await expect(scrapeWith(m)).rejects.toBeInstanceOf(EngineError);
+ const posts = fake.calls.filter(call => call.method === "POST");
+ expect(posts).toHaveLength(1);
+ return posts[0].body;
+ };
+
+ it("pin a US-only team to the US with country selection off", async () => {
+ const body = await sentRequest(
+ meta({
+ teamFlags: { usOnly: true },
+ location: { country: "us-generic", languages: ["en-US"] },
+ }),
+ );
+ expect(body.geolocation).toEqual({ country: "us", languages: ["en-US"] });
+ expect(body.behaviorOverrides).toEqual({ disableCountrySelection: true });
+ expect(body).not.toHaveProperty("usOnly");
+ });
+
+ it("pin a US-only team without a location", async () => {
+ const body = await sentRequest(meta({ teamFlags: { usOnly: true } }));
+ expect(body.geolocation).toEqual({ country: "us" });
+ expect(body.behaviorOverrides).toEqual({ disableCountrySelection: true });
+ });
+
+ it("keep Safe Mode's other overrides for a US-only team", async () => {
+ const body = await sentRequest(
+ meta({ teamFlags: { usOnly: true }, safeMode }),
+ );
+ expect(body.behaviorOverrides).toEqual({
+ disableSiteHandling: true,
+ exposeWebdriver: false,
+ useHeadlessUserAgent: false,
+ disablePlatformSelection: false,
+ disableCountrySelection: true,
+ disableAutomaticReferrer: true,
+ });
+ });
+
+ it.each([undefined, { usOnly: false }])(
+ "leave the request unchanged for team flags %j",
+ async teamFlags => {
+ const location = { country: "us-generic", languages: ["en-US"] };
+ const body = await sentRequest(meta({ teamFlags, location }));
+ expect(body.geolocation).toEqual(location);
+ expect(body).not.toHaveProperty("behaviorOverrides");
+ expect(body).not.toHaveProperty("usOnly");
+
+ fake.calls = [];
+ const withSafeMode = await sentRequest(
+ meta({ teamFlags, location, safeMode }),
+ );
+ expect(withSafeMode.behaviorOverrides.disableCountrySelection).toBe(
+ false,
+ );
+ },
+ );
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 7c28fe74c..a6b7e1740 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -7,7 +7,7 @@ import {
FireEngineScrapeRequestChromeCDP,
FireEngineScrapeRequestCommon,
FireEngineScrapeRequestTLSClient,
- safeModeParams,
+ requestOverrides,
} from "./scrape";
import { stripCredentialHeaders } from "../../../../lib/safe-mode";
import { EngineScrapeResult } from "..";
@@ -523,7 +523,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
name: meta.options.profile.name,
}
: undefined,
- ...safeModeParams(meta.internalOptions.safeMode),
+ ...requestOverrides(meta),
};
// Safe Mode worker-side hardening: neutralize anything that request-time
@@ -738,7 +738,7 @@ export async function scrapeURLWithFireEngineTLSClient(
!meta.internalOptions.zeroDataRetention &&
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
- ...safeModeParams(meta.internalOptions.safeMode),
+ ...requestOverrides(meta),
};
const sm = meta.internalOptions.safeMode;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index e3893aaba..a58f3e27d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -21,6 +21,7 @@ import {
} from "../../error";
import { Meta } from "../..";
import type { ResolvedSafeMode } from "../../../../lib/safe-mode";
+import { isUsOnly } from "../../../../lib/us-only";
import { config } from "../../../../config";
@@ -87,6 +88,24 @@ export function safeModeParams(
};
}
+/**
+ * Safe Mode's overrides, plus a pin for US-only teams: their fetches use the
+ * country "us" with fire-engine's own country selection turned off.
+ */
+export function requestOverrides(
+ meta: Meta,
+): Pick<FireEngineScrapeRequestCommon, "geolocation" | "behaviorOverrides"> {
+ const params = safeModeParams(meta.internalOptions.safeMode);
+ if (!isUsOnly(meta.internalOptions.teamFlags ?? undefined)) return params;
+ return {
+ geolocation: { ...meta.options.location, country: "us" },
+ behaviorOverrides: {
+ ...params.behaviorOverrides,
+ disableCountrySelection: true,
+ },
+ };
+}
+
export type FireEngineScrapeRequestChromeCDP = {
engine: "chrome-cdp";
skipTlsVerification?: boolean;
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 9a60d3525..1ecad12c7 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -131,6 +131,7 @@ import {
SAFE_MODE_LOGIN_ACTIONS,
} from "../../lib/safe-mode";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
import { loadFirstScreen } from "../../lib/branding/jev";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { canonicalizeUrl } from "../../lib/threat-protection/providers/web-risk/canonicalize";
@@ -1367,6 +1368,17 @@ export async function scrapeURL(
options.skipTlsVerification = false;
}
+ const usOnlyLocationError = usOnlyLocationRefusal(
+ internalOptions.teamFlags ?? undefined,
+ options.location?.country,
+ );
+ if (usOnlyLocationError) {
+ return {
+ success: false,
+ error: new UsOnlyUnavailableError(usOnlyLocationError),
+ };
+ }
+
const meta = await buildMetaObject(
id,
url,
@@ -1467,6 +1479,7 @@ export async function scrapeURL(
url: urlToCheck,
zeroDataRetention:
internalOptions.zeroDataRetention || false,
+ usOnly: isUsOnly(internalOptions.teamFlags ?? undefined),
location: options.location,
skipTlsVerification: options.skipTlsVerification,
},
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index e5ebe8949..5a3415674 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -1637,6 +1637,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
url: job.data.sitemapUrl,
maxAge: 48 * 60 * 60 * 1000,
zeroDataRetention: job.data.zeroDataRetention ?? false,
+ usOnly: crawler.usOnly,
location: job.data.location,
crawlId: job.data.crawl_id,
logger,