Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): pin US-only teams' page fetches to the US (#5594)
Gergő Móriczf9729cf88ac2
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 132b13901..102de5b1f 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -381,6 +381,17 @@ jobs:--reporter=defaultworking-directory: apps/api+ - name: Run US-only page fetch regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/lib/us-only.test.ts+ src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts+ src/scraper/scrapeURL/__tests__/usOnly.test.ts+ src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run country check route regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=defaultdiff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.tsindex 9e8b19aee..88c80ed34 100644--- a/apps/api/src/__tests__/snips/v2/us-only.test.ts+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts@@ -9,6 +9,8 @@ import {Identity,crawlStart,extractRaw,+ map,+ browserCreateRaw,} from "./lib";import { config } from "../../../config";@@ -200,6 +202,86 @@ describeIf(TEST_PRODUCTION)("US-only teams", () => {scrapeTimeout,);+ it.concurrent(+ "scrapes with a US location",+ async () => {+ const response = await scrape(+ { url: "https://firecrawl.dev", location: { country: "US" } },+ identity,+ );++ expect(response.markdown).toBeDefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses a scrape located outside the US",+ async () => {+ const response = await scrapeRaw(+ {+ url: "https://firecrawl.dev",+ location: { country: "NL" },+ timeout: scrapeTimeout,+ },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");+ expect(response.body.error).toContain(+ 'location.country "NL" isn\'t available to teams restricted to US-only processing',+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses a crawl located outside the US",+ async () => {+ const response = await crawlStart(+ {+ url: "https://firecrawl.dev",+ scrapeOptions: { location: { country: "NL" } },+ },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses a map located outside the US",+ async () => {+ const response = await map(+ { url: "https://firecrawl.dev", location: { country: "NL" } },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses a browser session located outside the US",+ async () => {+ const response = await browserCreateRaw(+ { location: { country: "NL" } },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");+ },+ scrapeTimeout,+ );+it.concurrent("refuses X/Twitter URLs",async () => {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 2188943b3..3117651b8 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -49,6 +49,7 @@ import {} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { requestCreditsShards } from "../../lib/request-credits-store";+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";export async function crawlController(req: Request, res: Response) {try {@@ -216,6 +217,17 @@ export async function crawlController(req: Request, res: Response) {});}+ const usOnlyLocationError = usOnlyLocationRefusal(+ auth.chunk?.flags ?? undefined,+ pageOptions?.geolocation?.country,+ );+ if (usOnlyLocationError) {+ return res.status(403).json({+ code: "US_ONLY_UNAVAILABLE",+ error: usOnlyLocationError,+ });+ }+// if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?// try {// const a = new WebScraperDataProvider();@@ -253,6 +265,9 @@ export async function crawlController(req: Request, res: Response) {);internalOptions.disableSmartWaitCache = true; // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterinternalOptions.orgId = auth.chunk?.org_id ?? null;+ if (isUsOnly(auth.chunk?.flags ?? undefined)) {+ internalOptions.teamFlags = auth.chunk?.flags;+ }internalOptions.saveScrapeResultToGCS = process.env.GCS_FIRE_ENGINE_BUCKET_NAME? truediff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 257da3ddb..e8941e40a 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -39,6 +39,7 @@ import {SAFE_MODE_V0_UNSUPPORTED_MESSAGE,} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";async function scrapeHelper(jobId: string,@@ -52,6 +53,7 @@ async function scrapeHelper(): Promise<{success: boolean;error?: string;+ code?: string;data?: V0Document | { url: string };returnCode: number;}> {@@ -88,6 +90,19 @@ async function scrapeHelper(};}+ const usOnlyLocationError = usOnlyLocationRefusal(+ acuc?.flags ?? undefined,+ pageOptions?.geolocation?.country,+ );+ if (usOnlyLocationError) {+ return {+ success: false,+ error: usOnlyLocationError,+ code: "US_ONLY_UNAVAILABLE",+ returnCode: 403,+ };+ }+const { scrapeOptions, internalOptions } = fromV0Combo(pageOptions,extractorOptions,@@ -97,6 +112,9 @@ async function scrapeHelper();internalOptions.orgId = acuc?.org_id ?? null;+ if (isUsOnly(acuc?.flags ?? undefined)) {+ internalOptions.teamFlags = acuc?.flags;+ }internalOptions.saveScrapeResultToGCS = process.env.GCS_FIRE_ENGINE_BUCKET_NAME? truediff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex e8c44d6ee..042038c7c 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -91,6 +91,9 @@ async function searchHelper(team_id,);internalOptions.orgId = org_id;+ if (isUsOnly(flags)) {+ internalOptions.teamFlags = flags;+ }if (justSearch) {const searchCredits = Math.ceil(res.length / 10) * 2;diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 4487c922c..750dfda77 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -46,7 +46,7 @@ import {} from "../../lib/keyless";import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";-import { isUsOnly } from "../../lib/us-only";+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";// Used for deep research@@ -200,6 +200,22 @@ export async function searchController(});}+ const usOnlyLocationError =+ requestedFormats.length > 0+ ? usOnlyLocationRefusal(+ req.acuc?.flags,+ req.body.scrapeOptions?.location?.country,+ req.body.scrapeOptions?.geolocation?.country,+ )+ : undefined;+ if (usOnlyLocationError) {+ return res.status(403).json({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: usOnlyLocationError,+ });+ }+// Search only scrapes (and only honors scrapeOptions) when formats are// requested, so the scrape-option checks apply only then.if (diff --git a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.tsindex f7474925a..9a1b2b2c2 100644--- a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts+++ b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts@@ -772,6 +772,84 @@ describe("scrapeInteractController", () => {expect(res.status).toHaveBeenCalledWith(status);expect(getHangarRecording).not.toHaveBeenCalled();});++ describe("US-only teams", () => {+ const usOnlyRequest = (body: Record<string, unknown>) =>+ ({+ body,+ headers: {},+ auth: { team_id: "team-123" },+ acuc: { flags: { usOnly: true } },+ }) as any;++ beforeEach(() => {+ vi.mocked(createHangarBrowser).mockResolvedValue({+ id: "br_us",+ max_expires_at: 700,+ cdp_url: "wss://hangar.example/cdp",+ } as any);+ vi.mocked(insertBrowserSession).mockImplementation(+ async row => row as any,+ );+ });++ it("refuses a browser session located outside the US", async () => {+ const res = buildRes();+ await browserCreateController(+ usOnlyRequest({ location: { country: "NL" } }),+ res,+ );+ expect(res.status).toHaveBeenCalledWith(403);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: expect.stringContaining('location.country "NL"'),+ });+ expect(createHangarBrowser).not.toHaveBeenCalled();+ });++ it("asks Hangar for a US-only exit", async () => {+ const res = buildRes();+ await browserCreateController(+ usOnlyRequest({ location: { country: "us" } }),+ res,+ );+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ success: true }),+ );+ expect(createHangarBrowser).toHaveBeenCalledWith(+ expect.any(String),+ "team-123",+ expect.objectContaining({ usOnly: true }),+ );+ });++ it("refuses to interact from a scrape located outside the US", async () => {+ vi.mocked(readScrapeJobState).mockResolvedValueOnce({+ status: "completed",+ requestId: "scrape-123",+ completedAtMs: Date.now(),+ creditsBilled: 1,+ replay: { targetUrl: "https://example.com", waitForMs: 0, actions: [] },+ browser: { blockAds: true, location: { country: "nl" } },+ } as any);+ const res = buildRes();+ await scrapeInteractController(+ {+ ...usOnlyRequest({ code: "console.log('ok')" }),+ params: { jobId: "scrape-123" },+ },+ res,+ );+ expect(res.status).toHaveBeenCalledWith(403);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: expect.stringContaining('location.country "NL"'),+ });+ expect(createHangarBrowser).not.toHaveBeenCalled();+ });+ });});vi.mock("../../../services/redlock", () => ({ redlock: { using: vi.fn() } }));diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex 7c1219aeb..cbc13b2f7 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -36,6 +36,7 @@ import {import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";import { browserProfileNameSchema } from "../../lib/browser-profiles";import { withZeroDataRetention } from "../../lib/otel-tracer";+import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";export const browserCreateRequestSchema = z.object({ttl: z.number().int().min(30).max(3600).default(600),@@ -94,6 +95,13 @@ export function browserError(signup_url: prompt.signup_url,});}+ if (error instanceof UsOnlyUnavailableError) {+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }const knownError =error instanceof BrowserSessionError || error instanceof HangarError;return res.status(knownError ? error.status : 502).json({diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 06465c149..7ca2d8686 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -521,6 +521,17 @@ async function createSessionForScrape(});return { session };} catch (error) {+ if (error instanceof UsOnlyUnavailableError) {+ return {+ error: true as const,+ status: 403,+ body: {+ success: false as const,+ code: error.code,+ error: error.message,+ },+ };+ }logger.error("Failed to initialize scrape browser session", { error });const knownError =error instanceof BrowserSessionError || error instanceof HangarError;diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 06ba0b523..aeb519786 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -70,7 +70,7 @@ import {autoToolsEnabled,queryInferenceEnabled,} from "../../search/tool-intent";-import { isUsOnly } from "../../lib/us-only";+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";import { UsOnlyUpstreamRefusalError } from "../../lib/research-upstream";import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";@@ -337,6 +337,21 @@ async function searchControllerInner(});}+ const usOnlyLocationError =+ requestedFormats.length > 0+ ? usOnlyLocationRefusal(+ req.acuc?.flags,+ req.body.scrapeOptions?.location?.country,+ )+ : undefined;+ if (usOnlyLocationError) {+ return res.status(403).json({+ success: false,+ code: "US_ONLY_UNAVAILABLE",+ error: usOnlyLocationError,+ });+ }+// Search only scrapes (and only honors scrapeOptions) when formats are// requested, so the scrape-option checks apply only then.if (diff --git a/apps/api/src/lib/browser-lifecycle.ts b/apps/api/src/lib/browser-lifecycle.tsindex f4746bc9a..de2e035f1 100644--- a/apps/api/src/lib/browser-lifecycle.ts+++ b/apps/api/src/lib/browser-lifecycle.ts@@ -55,6 +55,8 @@ import { getScrapeZDR } from "./zdr-helpers";import { withZeroDataRetention } from "./otel-tracer";import { redlock } from "../services/redlock";import { redisRateLimitClient } from "../services/rate-limiter";+import { isUsOnly, usOnlyLocationRefusal } from "./us-only";+import { UsOnlyUnavailableError } from "../scraper/scrapeURL/error";const logger = rootLogger;@@ -157,6 +159,12 @@ export async function createBrowserSession(initialize?: (browserId: string) => Promise<void>;},) {+ const usOnlyLocationError = usOnlyLocationRefusal(+ req.acuc?.flags,+ options.location?.country,+ );+ if (usOnlyLocationError)+ throw new UsOnlyUnavailableError(usOnlyLocationError);const zeroDataRetention =getBrowserZDR(req) || options.zeroDataRetention === true;if (zeroDataRetention && (options.recordSession || options.profile)) {@@ -242,6 +250,7 @@ async function createBrowserSessionInternal(const browser = await createHangarBrowser(id, req.auth.team_id, {...options,zeroDataRetention,+ usOnly: isUsOnly(req.acuc?.flags),});browserId = browser.id;if (options.initialize) await options.initialize(browser.id);diff --git a/apps/api/src/lib/browser-options.test.ts b/apps/api/src/lib/browser-options.test.tsindex 900b56620..369670900 100644--- a/apps/api/src/lib/browser-options.test.ts+++ b/apps/api/src/lib/browser-options.test.ts@@ -13,6 +13,7 @@ import { createHangarBrowser } from "./hangar";async function hangarCreateBody(options: BrowserOptions,zeroDataRetention = false,+ usOnly = false,) {const fetchMock = vi.fn(async (_url: string, _init: RequestInit) =>Response.json({@@ -29,6 +30,7 @@ async function hangarCreateBody(streamWebView: true,recordSession: false,zeroDataRetention,+ usOnly,});return JSON.parse(fetchMock.mock.calls[0][1].body as string);}@@ -137,6 +139,39 @@ describe("Hangar create body under zero data retention", () => {});});+describe("Hangar create body for US-only teams", () => {+ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("always asks for a US exit", async () => {+ expect(+ (await hangarCreateBody({ blockAds: true }, false, true)).proxy,+ ).toEqual({+ country: "us",+ type: "basic",+ });+ expect(+ (+ await hangarCreateBody(+ browserOptionsFromScrape({ proxy: "stealth" }),+ false,+ true,+ )+ ).proxy,+ ).toEqual({ country: "us", type: "mobile" });+ });++ it("keeps ZDR exits within the US", async () => {+ const body = await hangarCreateBody({ blockAds: true }, true, true);+ expect(body.proxy).toEqual({+ country: "us",+ type: "basic",+ zero_data_retention: true,+ });+ });+});+describe("isBrowserOptions", () => {it("accepts options persisted before location and proxy were stored", () => {expect(isBrowserOptions({ blockAds: true })).toBe(true);diff --git a/apps/api/src/lib/browser-options.ts b/apps/api/src/lib/browser-options.tsindex d36b33e97..96e9ddf24 100644--- a/apps/api/src/lib/browser-options.ts+++ b/apps/api/src/lib/browser-options.ts@@ -55,6 +55,7 @@ export function isBrowserOptions(value: unknown): value is BrowserOptions {export function hangarBrowserOptions(options: BrowserOptions,zeroDataRetention: boolean,+ usOnly = false,) {return {// uBlock Origin Lite covers ads, trackers and cookie notices.@@ -69,11 +70,19 @@ export function hangarBrowserOptions(: {}),// Scrape's stealth proxy is Hangar's mobile pool. Omitted fields keep// Hangar's defaults: country "us", type "basic". Under ZDR, Hangar picks- // only ZDR exits and fails the browser if none is available.- ...(options.location || options.proxy === "stealth" || zeroDataRetention+ // only ZDR exits and fails the browser if none is available. A US-only+ // team always asks for a US exit.+ ...(options.location ||+ options.proxy === "stealth" ||+ zeroDataRetention ||+ usOnly? {proxy: {- ...(options.location ? { country: options.location.country } : {}),+ ...(usOnly+ ? { country: "us" }+ : options.location+ ? { country: options.location.country }+ : {}),type: options.proxy === "stealth" ? "mobile" : "basic",...(zeroDataRetention ? { zero_data_retention: true } : {}),},diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 9bf1becd1..c8b3ca7ee 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -9,6 +9,7 @@ import type { Logger } from "winston";import { withSpan, setSpanAttributes } from "./otel-tracer";import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";import { resolveSafeMode } from "./safe-mode";+import { isUsOnly } from "./us-only";import {firstPipelineError,reportPipelineError,@@ -1060,6 +1061,7 @@ export function crawlToCrawler(currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,zeroDataRetention:(getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,+ usOnly: isUsOnly(sc.internalOptions?.teamFlags) || isUsOnly(teamFlags),location: sc.scrapeOptions?.location,headers: sc.scrapeOptions?.headers,robotsUserAgent: sc.crawlerOptions?.robotsUserAgent,diff --git a/apps/api/src/lib/hangar.ts b/apps/api/src/lib/hangar.tsindex 0550ab570..580bf3660 100644--- a/apps/api/src/lib/hangar.ts+++ b/apps/api/src/lib/hangar.ts@@ -113,6 +113,7 @@ export async function createHangarBrowser(streamWebView: boolean;recordSession: boolean;zeroDataRetention: boolean;+ usOnly?: boolean;},): Promise<HangarCreated> {const body = {@@ -125,7 +126,7 @@ export async function createHangarBrowser(},recording: { enabled: options.recordSession },execution: { enabled: true },- ...hangarBrowserOptions(options, options.zeroDataRetention),+ ...hangarBrowserOptions(options, options.zeroDataRetention, options.usOnly),};let created: HangarCreated | undefined;for (let attempt = 0; attempt < 3; attempt++) {diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex 69e72efec..6eb1afd52 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -16,6 +16,7 @@ import {THREAT_PROTECTION_NOT_ENABLED_MESSAGE,THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,} from "./threat-protection/request";+import { usOnlyLocationRefusal } from "./us-only";type LocationOptions = { country?: string };type ThreatProtectionOption = { mode?: string };@@ -23,8 +24,10 @@ type ThreatProtectionOption = { mode?: string };interface APIRequest {zeroDataRetention?: boolean;location?: LocationOptions;+ geolocation?: LocationOptions;scrapeOptions?: {location?: LocationOptions;+ geolocation?: LocationOptions;threatProtection?: ThreatProtectionOption;};crawlerOptions?: {@@ -171,6 +174,17 @@ export function checkPermissions(}}+ const usOnlyLocationError = usOnlyLocationRefusal(+ flags,+ request.location?.country,+ request.geolocation?.country,+ request.scrapeOptions?.location?.country,+ request.scrapeOptions?.geolocation?.country,+ );+ if (usOnlyLocationError) {+ return { error: usOnlyLocationError, code: "US_ONLY_UNAVAILABLE" };+ }+// ip whitelist permsconst needsWhitelist =request.location?.country === "us-whitelist" ||diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex d27d6d59d..2fb037c4f 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -19,6 +19,7 @@ export async function fetchRobotsTxt({url,zeroDataRetention,+ usOnly,location,headers,skipCache,@@ -26,6 +27,7 @@ export async function fetchRobotsTxt(}: {url: string;zeroDataRetention: boolean;+ usOnly?: boolean;skipTlsVerification?: boolean;location?: ScrapeOptions["location"];headers?: Record<string, string>;@@ -90,6 +92,7 @@ export async function fetchRobotsTxt(: undefined,teamId: "robots-txt",zeroDataRetention,+ ...(usOnly ? { teamFlags: { usOnly: true } } : {}),},new CostTracking(),);diff --git a/apps/api/src/lib/us-only.test.ts b/apps/api/src/lib/us-only.test.tsnew file mode 100644index 000000000..9b7cd69ac--- /dev/null+++ b/apps/api/src/lib/us-only.test.ts@@ -0,0 +1,82 @@+import { checkPermissions } from "./permissions";+import { usOnlyLocationRefusal } from "./us-only";++const usOnly = { usOnly: true };++describe("usOnlyLocationRefusal", () => {+ it("refuses a country outside the US for a US-only team", () => {+ expect(usOnlyLocationRefusal(usOnly, "nl")).toBe(+ 'location.country "NL" isn\'t available to teams restricted to US-only processing, because pages are fetched only from US exits. Omit location or set it to "US".',+ );+ });++ it.each(["us", "US", "us-generic", "US-generic", "", null, 42, undefined])(+ "allows %s",+ country => {+ expect(usOnlyLocationRefusal(usOnly, country)).toBeUndefined();+ },+ );++ it("refuses us-whitelist, which isn't the US", () => {+ expect(usOnlyLocationRefusal(usOnly, "us-whitelist")).toContain(+ 'location.country "US-WHITELIST"',+ );+ });++ it("names the first country outside the US", () => {+ expect(+ usOnlyLocationRefusal(usOnly, "us", undefined, "fr", "de"),+ ).toContain('"FR"');+ });++ it("leaves teams without the restriction alone", () => {+ expect(usOnlyLocationRefusal(undefined, "nl")).toBeUndefined();+ expect(usOnlyLocationRefusal(null, "nl")).toBeUndefined();+ expect(usOnlyLocationRefusal({ usOnly: false }, "nl")).toBeUndefined();+ });+});++describe("checkPermissions for US-only teams", () => {+ it.each([+ ["location", { location: { country: "nl" } }],+ ["v1 geolocation", { geolocation: { country: "NL" } }],+ [+ "scrapeOptions.location",+ { scrapeOptions: { location: { country: "nl" } } },+ ],+ [+ "scrapeOptions.geolocation",+ { scrapeOptions: { geolocation: { country: "NL" } } },+ ],+ ])("refuses a non-US %s with US_ONLY_UNAVAILABLE", (_, request) => {+ const result = checkPermissions(request, usOnly);+ expect(result.code).toBe("US_ONLY_UNAVAILABLE");+ expect(result.error).toContain('location.country "NL"');+ });++ it("refuses us-whitelist with US_ONLY_UNAVAILABLE", () => {+ expect(+ checkPermissions({ location: { country: "us-whitelist" } }, usOnly).code,+ ).toBe("US_ONLY_UNAVAILABLE");+ });++ it("allows a US location or none", () => {+ expect(checkPermissions({ location: { country: "us" } }, usOnly)).toEqual(+ {},+ );+ expect(+ checkPermissions({ location: { country: "us-generic" } }, usOnly),+ ).toEqual({});+ expect(+ checkPermissions({ geolocation: { country: "US-generic" } }, usOnly),+ ).toEqual({});+ expect(checkPermissions({}, usOnly)).toEqual({});+ });++ it("leaves other teams' locations alone", () => {+ expect(checkPermissions({ location: { country: "nl" } }, null)).toEqual({});+ expect(+ checkPermissions({ location: { country: "nl" } }, { usOnly: false }),+ ).toEqual({});+ });+});diff --git a/apps/api/src/lib/us-only.ts b/apps/api/src/lib/us-only.tsindex 10f924639..8bd245ca7 100644--- a/apps/api/src/lib/us-only.ts+++ b/apps/api/src/lib/us-only.ts@@ -5,6 +5,29 @@ export function isUsOnly(flags: TeamFlags | undefined): boolean {return flags?.usOnly === true;}+const US_COUNTRIES = ["us", "us-generic"];++/**+ * The refusal for a US-only team asking for a country other than the US, since+ * its pages are fetched only from US exits. Undefined for an unrestricted team+ * or when every country given is "us" or "us-generic" (the schema default).+ */+export function usOnlyLocationRefusal(+ flags: TeamFlags | undefined,+ ...countries: unknown[]+): string | undefined {+ if (!isUsOnly(flags)) return undefined;+ const country = countries.find(+ (value): value is string =>+ typeof value === "string" &&+ value !== "" &&+ !US_COUNTRIES.includes(value.toLowerCase()),+ );+ return country === undefined+ ? undefined+ : `location.country "${country.toUpperCase()}" isn't available to teams restricted to US-only processing, because pages are fetched only from US exits. Omit location or set it to "US".`;+}+/** The `data_region` value that pins a US-only team's GPU work to US capacity. */export const US_DATA_REGION = "us";diff --git a/apps/api/src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts b/apps/api/src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.tsnew file mode 100644index 000000000..c36e35d2d--- /dev/null+++ b/apps/api/src/scraper/WebScraper/__tests__/usOnlyDiscovery.test.ts@@ -0,0 +1,128 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++vi.mock("../../scrapeURL", () => ({+ scrapeURL: vi.fn(async () => ({ success: false, error: "unreachable" })),+}));++import { scrapeURL } from "../../scrapeURL";+import { crawlToCrawler } from "../../../lib/crawl-redis";+import { getLinksFromSitemap } from "../sitemap";+import { scrapeSitemap } from "../../crawler/sitemap";+import { logger } from "../../../lib/logger";++const storedCrawl = (teamFlags?: { usOnly?: boolean }) =>+ ({+ originUrl: "https://example.com/",+ crawlerOptions: {},+ scrapeOptions: {},+ internalOptions: { teamId: "team-a", orgId: null, teamFlags },+ team_id: "team-a",+ createdAt: 0,+ }) as any;++const internalOptionsSent = () =>+ vi+ .mocked(scrapeURL)+ .mock.calls.map(([, , , internalOptions]) => internalOptions);++beforeEach(() => {+ vi.mocked(scrapeURL).mockClear();+});++describe("crawl discovery for US-only teams", () => {+ it("marks the crawler US-only from the stored crawl or the team's live flags", () => {+ expect(+ crawlToCrawler("crawl-1", storedCrawl({ usOnly: true }), null).usOnly,+ ).toBe(true);+ expect(+ crawlToCrawler("crawl-1", storedCrawl(), { usOnly: true }).usOnly,+ ).toBe(true);+ expect(crawlToCrawler("crawl-1", storedCrawl(), null).usOnly).toBe(false);+ });++ it("fetches robots.txt as US-only", async () => {+ await crawlToCrawler(+ "crawl-1",+ storedCrawl({ usOnly: true }),+ null,+ ).getRobotsTxt();+ expect(internalOptionsSent()).toEqual([+ expect.objectContaining({+ teamId: "robots-txt",+ teamFlags: { usOnly: true },+ }),+ ]);+ });++ it("fetches other teams' robots.txt without team flags", async () => {+ await crawlToCrawler("crawl-1", storedCrawl(), null).getRobotsTxt();+ expect(internalOptionsSent()).toHaveLength(1);+ expect(internalOptionsSent()[0]).not.toHaveProperty("teamFlags");+ });++ it("fetches nested sitemaps as US-only", async () => {+ vi.mocked(scrapeURL).mockResolvedValueOnce({+ success: true,+ document: {+ rawHtml:+ '<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>https://example.com/sitemap-posts.xml</loc></sitemap></sitemapindex>',+ metadata: { statusCode: 200 },+ },+ } as any);+ await getLinksFromSitemap(+ {+ sitemapUrl: "https://example.com/sitemap.xml",+ urlsHandler: () => {},+ mode: "fire-engine",+ zeroDataRetention: false,+ usOnly: true,+ },+ logger,+ "crawl-1",+ new Set(),+ );+ expect(vi.mocked(scrapeURL).mock.calls.map(([, url]) => url)).toEqual([+ "https://example.com/sitemap.xml",+ "https://example.com/sitemap-posts.xml",+ ]);+ expect(internalOptionsSent()).toEqual([+ expect.objectContaining({ teamFlags: { usOnly: true } }),+ expect.objectContaining({ teamFlags: { usOnly: true } }),+ ]);+ });++ it("fetches sitemaps as US-only", async () => {+ await getLinksFromSitemap(+ {+ sitemapUrl: "https://example.com/sitemap.xml",+ urlsHandler: () => {},+ mode: "fire-engine",+ zeroDataRetention: false,+ usOnly: true,+ },+ logger,+ "crawl-1",+ new Set(),+ );+ await expect(+ scrapeSitemap({+ url: "https://example.com/sitemap-2.xml",+ maxAge: 0,+ zeroDataRetention: false,+ usOnly: true,+ location: undefined,+ crawlId: "crawl-1",+ }),+ ).rejects.toThrow();+ expect(internalOptionsSent()).toEqual([+ expect.objectContaining({+ teamId: "sitemap",+ teamFlags: { usOnly: true },+ }),+ expect.objectContaining({+ teamId: "sitemap",+ teamFlags: { usOnly: true },+ }),+ ]);+ });+});diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 980f3fe85..fe51024ca 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -85,6 +85,7 @@ export class WebCrawler {private maxDiscoveryDepth: number | undefined;private currentDiscoveryDepth: number;private zeroDataRetention: boolean;+ public readonly usOnly: boolean;private location?: ScrapeOptions["location"];private headers?: Record<string, string>;private robotsUserAgent?: string;@@ -109,6 +110,7 @@ export class WebCrawler {maxDiscoveryDepth,currentDiscoveryDepth,zeroDataRetention,+ usOnly,location,headers,robotsUserAgent,@@ -131,6 +133,7 @@ export class WebCrawler {maxDiscoveryDepth?: number;currentDiscoveryDepth?: number;zeroDataRetention?: boolean;+ usOnly?: boolean;location?: ScrapeOptions["location"];headers?: Record<string, string>;robotsUserAgent?: string;@@ -155,6 +158,7 @@ export class WebCrawler {this.ignoreRobotsTxt = ignoreRobotsTxt ?? false;this.regexOnFullURL = regexOnFullURL ?? false;this.zeroDataRetention = zeroDataRetention ?? false;+ this.usOnly = usOnly ?? false;this.logger = _logger.child({crawlId: this.jobId,module: "WebCrawler",@@ -500,6 +504,7 @@ export class WebCrawler {url: this.robotsTxtUrl,zeroDataRetention: this.zeroDataRetention,location: this.location,+ usOnly: this.usOnly,skipTlsVerification,...(this.robotsUserAgent? {@@ -941,6 +946,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ usOnly: this.usOnly,headers: this.headers,},this.logger,@@ -1016,6 +1022,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ usOnly: this.usOnly,headers: this.headers,},this.logger,@@ -1060,6 +1067,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ usOnly: this.usOnly,headers: this.headers,},this.logger,@@ -1088,6 +1096,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ usOnly: this.usOnly,headers: this.headers,},this.logger,diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 198ba4cdc..64b0be975 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -28,6 +28,7 @@ export async function getLinksFromSitemap(mode = "axios",maxAge = 0,zeroDataRetention,+ usOnly,location,headers,}: {@@ -36,6 +37,7 @@ export async function getLinksFromSitemap(mode?: "axios" | "fire-engine";maxAge?: number;zeroDataRetention: boolean;+ usOnly?: boolean;location?: ScrapeOptions["location"];headers?: Record<string, string>;},@@ -126,6 +128,7 @@ export async function getLinksFromSitemap(: undefined,teamId: "sitemap",zeroDataRetention,+ ...(usOnly ? { teamFlags: { usOnly: true } } : {}),},new CostTracking(),);@@ -213,6 +216,7 @@ export async function getLinksFromSitemap(urlsHandler,mode,zeroDataRetention,+ usOnly,location,headers,maxAge,@@ -246,6 +250,7 @@ export async function getLinksFromSitemap(urlsHandler,mode,zeroDataRetention,+ usOnly,location,headers,maxAge,@@ -295,6 +300,7 @@ export async function getLinksFromSitemap(urlsHandler,mode,zeroDataRetention,+ usOnly,location,headers,maxAge,diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex f96fac663..9e6b681b4 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -21,6 +21,7 @@ type SitemapScrapeOptions = {url: string;maxAge: number;zeroDataRetention: boolean;+ usOnly?: boolean;location: ScrapeOptions["location"];crawlId: string;logger?: Logger;@@ -101,6 +102,7 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {teamId: "sitemap",orgId: null,zeroDataRetention: options.zeroDataRetention,+ ...(options.usOnly ? { teamFlags: { usOnly: true } } : {}),crawlId: options.crawlId,isPreCrawl: options.isPreCrawl,},diff --git a/apps/api/src/scraper/scrapeURL/__tests__/usOnly.test.ts b/apps/api/src/scraper/scrapeURL/__tests__/usOnly.test.tsnew file mode 100644index 000000000..ec5881e9b--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/__tests__/usOnly.test.ts@@ -0,0 +1,46 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++vi.mock("../engines", async importOriginal => ({+ ...(await importOriginal<typeof import("../engines")>()),+ buildFallbackList: vi.fn(),+ scrapeURLWithEngine: vi.fn(),+}));++import { scrapeURL } from "..";+import { buildFallbackList, scrapeURLWithEngine } from "../engines";+import { UsOnlyUnavailableError } from "../error";+import { scrapeOptions } from "../../../controllers/v2/types";+import { CostTracking } from "../../../lib/cost-tracking";++const usOnlyTeam = {+ teamId: "team-a",+ orgId: null,+ teamFlags: { usOnly: true },+};++beforeEach(() => {+ vi.mocked(buildFallbackList).mockReset();+ vi.mocked(scrapeURLWithEngine).mockReset();+});++describe("scrapeURL for US-only teams", () => {+ it("refuses a location outside the US before any engine runs", async () => {+ const result = await scrapeURL(+ "us-only-location",+ "https://example.com",+ scrapeOptions.parse({ location: { country: "NL" } }),+ { ...usOnlyTeam },+ new CostTracking(),+ );++ expect(result.success).toBe(false);+ expect(!result.success && result.error).toBeInstanceOf(+ UsOnlyUnavailableError,+ );+ expect(!result.success && result.error.message).toContain(+ 'location.country "NL"',+ );+ expect(buildFallbackList).not.toHaveBeenCalled();+ expect(scrapeURLWithEngine).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.tsnew file mode 100644index 000000000..4229c87a8--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/usOnly.test.ts@@ -0,0 +1,148 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const fake = vi.hoisted(() => ({ calls: [] as any[] }));++vi.mock("../../../lib/fetch", () => ({+ robustFetch: vi.fn(async (params: any) => {+ fake.calls.push(params);+ return { error: "Scrape failed" };+ }),+ redactRequestBodyForLog: (body: unknown) => body,+}));++vi.mock("../../../../../lib/gcs-jobs", () => ({+ getDocFromGCS: vi.fn(async () => null),+}));++import { EngineError } from "../../../error";+import {+ scrapeURLWithFireEngineChromeCDP,+ scrapeURLWithFireEngineTLSClient,+} from "../index";+import type { ResolvedSafeMode } from "../../../../../lib/safe-mode";++const logger = {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child() {+ return this;+ },+} as any;++const safeMode: ResolvedSafeMode = {+ lockdown: false,+ domainControls: false,+ enforceRobots: false,+ disableStealthProxy: false,+ disableAuthentication: false,+ disableSiteHandling: true,+ exposeWebdriver: false,+ useHeadlessUserAgent: false,+ disablePlatformSelection: false,+ disableCountrySelection: false,+ disableAutomaticReferrer: true,+};++function meta({+ teamFlags,+ location,+ safeMode,+}: {+ teamFlags?: { usOnly?: boolean };+ location?: { country?: string; languages?: string[] };+ safeMode?: ResolvedSafeMode;+}) {+ return {+ id: "scrape-1",+ url: "https://example.com",+ options: {+ formats: [{ type: "markdown" }],+ skipTlsVerification: false,+ proxy: "auto",+ location,+ },+ internalOptions: {+ teamId: "team-a",+ zeroDataRetention: false,+ teamFlags,+ safeMode,+ },+ logger,+ abort: {+ scrapeTimeout: () => 30000,+ asSignal: () => undefined,+ throwIfAborted: () => {},+ },+ featureFlags: new Set(),+ mock: null,+ } as any;+}++beforeEach(() => {+ fake.calls = [];+});++describe.each([+ ["chrome-cdp", scrapeURLWithFireEngineChromeCDP],+ ["tlsclient", scrapeURLWithFireEngineTLSClient],+] as const)("fire-engine %s requests", (_, scrapeWith) => {+ const sentRequest = async (m: any) => {+ await expect(scrapeWith(m)).rejects.toBeInstanceOf(EngineError);+ const posts = fake.calls.filter(call => call.method === "POST");+ expect(posts).toHaveLength(1);+ return posts[0].body;+ };++ it("pin a US-only team to the US with country selection off", async () => {+ const body = await sentRequest(+ meta({+ teamFlags: { usOnly: true },+ location: { country: "us-generic", languages: ["en-US"] },+ }),+ );+ expect(body.geolocation).toEqual({ country: "us", languages: ["en-US"] });+ expect(body.behaviorOverrides).toEqual({ disableCountrySelection: true });+ expect(body).not.toHaveProperty("usOnly");+ });++ it("pin a US-only team without a location", async () => {+ const body = await sentRequest(meta({ teamFlags: { usOnly: true } }));+ expect(body.geolocation).toEqual({ country: "us" });+ expect(body.behaviorOverrides).toEqual({ disableCountrySelection: true });+ });++ it("keep Safe Mode's other overrides for a US-only team", async () => {+ const body = await sentRequest(+ meta({ teamFlags: { usOnly: true }, safeMode }),+ );+ expect(body.behaviorOverrides).toEqual({+ disableSiteHandling: true,+ exposeWebdriver: false,+ useHeadlessUserAgent: false,+ disablePlatformSelection: false,+ disableCountrySelection: true,+ disableAutomaticReferrer: true,+ });+ });++ it.each([undefined, { usOnly: false }])(+ "leave the request unchanged for team flags %j",+ async teamFlags => {+ const location = { country: "us-generic", languages: ["en-US"] };+ const body = await sentRequest(meta({ teamFlags, location }));+ expect(body.geolocation).toEqual(location);+ expect(body).not.toHaveProperty("behaviorOverrides");+ expect(body).not.toHaveProperty("usOnly");++ fake.calls = [];+ const withSafeMode = await sentRequest(+ meta({ teamFlags, location, safeMode }),+ );+ expect(withSafeMode.behaviorOverrides.disableCountrySelection).toBe(+ false,+ );+ },+ );+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 7c28fe74c..a6b7e1740 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -7,7 +7,7 @@ import {FireEngineScrapeRequestChromeCDP,FireEngineScrapeRequestCommon,FireEngineScrapeRequestTLSClient,- safeModeParams,+ requestOverrides,} from "./scrape";import { stripCredentialHeaders } from "../../../../lib/safe-mode";import { EngineScrapeResult } from "..";@@ -523,7 +523,7 @@ export async function scrapeURLWithFireEngineChromeCDP(name: meta.options.profile.name,}: undefined,- ...safeModeParams(meta.internalOptions.safeMode),+ ...requestOverrides(meta),};// Safe Mode worker-side hardening: neutralize anything that request-time@@ -738,7 +738,7 @@ export async function scrapeURLWithFireEngineTLSClient(!meta.internalOptions.zeroDataRetention &&meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,- ...safeModeParams(meta.internalOptions.safeMode),+ ...requestOverrides(meta),};const sm = meta.internalOptions.safeMode;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex e3893aaba..a58f3e27d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -21,6 +21,7 @@ import {} from "../../error";import { Meta } from "../..";import type { ResolvedSafeMode } from "../../../../lib/safe-mode";+import { isUsOnly } from "../../../../lib/us-only";import { config } from "../../../../config";@@ -87,6 +88,24 @@ export function safeModeParams(};}+/**+ * Safe Mode's overrides, plus a pin for US-only teams: their fetches use the+ * country "us" with fire-engine's own country selection turned off.+ */+export function requestOverrides(+ meta: Meta,+): Pick<FireEngineScrapeRequestCommon, "geolocation" | "behaviorOverrides"> {+ const params = safeModeParams(meta.internalOptions.safeMode);+ if (!isUsOnly(meta.internalOptions.teamFlags ?? undefined)) return params;+ return {+ geolocation: { ...meta.options.location, country: "us" },+ behaviorOverrides: {+ ...params.behaviorOverrides,+ disableCountrySelection: true,+ },+ };+}+export type FireEngineScrapeRequestChromeCDP = {engine: "chrome-cdp";skipTlsVerification?: boolean;diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 9a60d3525..1ecad12c7 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -131,6 +131,7 @@ import {SAFE_MODE_LOGIN_ACTIONS,} from "../../lib/safe-mode";import { resolveThreatProtection } from "../../lib/threat-protection/request";+import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";import { loadFirstScreen } from "../../lib/branding/jev";import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { canonicalizeUrl } from "../../lib/threat-protection/providers/web-risk/canonicalize";@@ -1367,6 +1368,17 @@ export async function scrapeURL(options.skipTlsVerification = false;}+ const usOnlyLocationError = usOnlyLocationRefusal(+ internalOptions.teamFlags ?? undefined,+ options.location?.country,+ );+ if (usOnlyLocationError) {+ return {+ success: false,+ error: new UsOnlyUnavailableError(usOnlyLocationError),+ };+ }+const meta = await buildMetaObject(id,url,@@ -1467,6 +1479,7 @@ export async function scrapeURL(url: urlToCheck,zeroDataRetention:internalOptions.zeroDataRetention || false,+ usOnly: isUsOnly(internalOptions.teamFlags ?? undefined),location: options.location,skipTlsVerification: options.skipTlsVerification,},diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e5ebe8949..5a3415674 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -1637,6 +1637,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {url: job.data.sitemapUrl,maxAge: 48 * 60 * 60 * 1000,zeroDataRetention: job.data.zeroDataRetention ?? false,+ usOnly: crawler.usOnly,location: job.data.location,crawlId: job.data.crawl_id,logger,