Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): check the static IP entitlement on search, extract and v0 (#5649)
tomsideguide4ce1241b0686
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 3117651b8..2d225b3a7 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -50,6 +50,7 @@ import {import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { requestCreditsShards } from "../../lib/request-credits-store";import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";+import { ipWhitelistRefusal } from "../../lib/permissions";export async function crawlController(req: Request, res: Response) {try {@@ -228,6 +229,14 @@ export async function crawlController(req: Request, res: Response) {});}+ const ipWhitelistError = ipWhitelistRefusal(+ auth.chunk?.flags,+ pageOptions?.geolocation?.country,+ );+ if (ipWhitelistError) {+ return res.status(403).json({ error: ipWhitelistError });+ }+// if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?// try {// const a = new WebScraperDataProvider();diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex e8941e40a..47ac8295e 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -40,6 +40,7 @@ import {} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";+import { ipWhitelistRefusal } from "../../lib/permissions";async function scrapeHelper(jobId: string,@@ -103,6 +104,18 @@ async function scrapeHelper(};}+ const ipWhitelistError = ipWhitelistRefusal(+ acuc?.flags,+ pageOptions?.geolocation?.country,+ );+ if (ipWhitelistError) {+ return {+ success: false,+ error: ipWhitelistError,+ returnCode: 403,+ };+ }+const { scrapeOptions, internalOptions } = fromV0Combo(pageOptions,extractorOptions,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 042038c7c..f759a0199 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -33,6 +33,7 @@ import {} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { isUsOnly } from "../../lib/us-only";+import { ipWhitelistRefusal } from "../../lib/permissions";import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";async function searchHelper(@@ -275,6 +276,14 @@ export async function searchController(req: Request, res: Response) {};const origin = req.body.origin ?? "api";+ const ipWhitelistError =+ pageOptions.fetchPageContent !== false+ ? ipWhitelistRefusal(chunk?.flags, pageOptions?.geolocation?.country)+ : undefined;+ if (ipWhitelistError) {+ return res.status(403).json({ error: ipWhitelistError });+ }+const searchOptions = req.body.searchOptions ?? { limit: 5 };try {diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 77906b59c..b3764ca31 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -36,6 +36,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { UnsupportedSiteError } from "../../lib/error";+import { ipWhitelistRefusal } from "../../lib/permissions";import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";import { isSdkOrigin } from "../../lib/sdk-origin";async function oldExtract(@@ -141,6 +142,18 @@ export async function extractController(});}+ const ipWhitelistError = ipWhitelistRefusal(+ req.acuc?.flags,+ req.body.scrapeOptions?.location?.country,+ req.body.scrapeOptions?.geolocation?.country,+ );+ if (ipWhitelistError) {+ return res.status(403).json({+ success: false,+ error: ipWhitelistError,+ });+ }+const invalidURLs: string[] =req.body.urls?.filter((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 750dfda77..40c258996 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -28,7 +28,7 @@ import {resolveSafeMode,getEffectiveSearchForcedKind,} from "../../lib/safe-mode";-import { checkPermissions } from "../../lib/permissions";+import { checkPermissions, ipWhitelistRefusal } from "../../lib/permissions";import {DocumentWithCostTracking,scrapeSearchResults,@@ -216,6 +216,21 @@ export async function searchController(});}+ const ipWhitelistError =+ requestedFormats.length > 0+ ? ipWhitelistRefusal(+ req.acuc?.flags,+ req.body.scrapeOptions?.location?.country,+ req.body.scrapeOptions?.geolocation?.country,+ )+ : undefined;+ if (ipWhitelistError) {+ return res.status(403).json({+ success: false,+ error: ipWhitelistError,+ });+ }+// Search only scrapes (and only honors scrapeOptions) when formats are// requested, so the scrape-option checks apply only then.if (diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 6f37af49b..530e99e09 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -24,6 +24,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { UnsupportedSiteError } from "../../lib/error";+import { ipWhitelistRefusal } from "../../lib/permissions";import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";import { checkExtractAvailable } from "../../lib/extract/fire-0/extraction-service-f0";@@ -60,6 +61,17 @@ export async function extractController(});}+ const ipWhitelistError = ipWhitelistRefusal(+ req.acuc?.flags,+ req.body.scrapeOptions?.location?.country,+ );+ if (ipWhitelistError) {+ return res.status(403).json({+ success: false,+ error: ipWhitelistError,+ });+ }+const extractId = uuidv7();const createdAt = Date.now();_logger.info("Extract starting...", {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex ffe06a9be..6165998f7 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -47,7 +47,7 @@ import {isLockdownZeroDataRetention,getEffectiveSearchForcedKind,} from "../../lib/safe-mode";-import { checkPermissions } from "../../lib/permissions";+import { checkPermissions, ipWhitelistRefusal } from "../../lib/permissions";import {actionTypesOf,checkKeyEndpointRestriction,@@ -334,6 +334,20 @@ async function searchControllerInner(});}+ const ipWhitelistError =+ requestedFormats.length > 0+ ? ipWhitelistRefusal(+ req.acuc?.flags,+ req.body.scrapeOptions?.location?.country,+ )+ : undefined;+ if (ipWhitelistError) {+ return res.status(403).json({+ success: false,+ error: ipWhitelistError,+ });+ }+// Search only scrapes (and only honors scrapeOptions) when formats are// requested, so the scrape-option checks apply only then.if (diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex 6eb1afd52..8ba1ee832 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -55,6 +55,20 @@ interface PermissionOptions {const SUPPORT_EMAIL = "[email protected]";+/** The refusal for static-IP egress ("us-whitelist") without the entitlement. */+export function ipWhitelistRefusal(+ flags: TeamFlags | null | undefined,+ ...countries: unknown[]+): string | undefined {+ const needsWhitelist = countries.some(+ value =>+ typeof value === "string" && value.toLowerCase() === "us-whitelist",+ );+ return needsWhitelist && !flags?.ipWhitelist+ ? `Static IP addresses are not enabled for your team. Contact ${SUPPORT_EMAIL} to get a dedicated set of IP addresses you can whitelist.`+ : undefined;+}+export function checkPermissions(request: APIRequest,flags?: TeamFlags,@@ -185,15 +199,13 @@ export function checkPermissions(return { error: usOnlyLocationError, code: "US_ONLY_UNAVAILABLE" };}- // ip whitelist perms- const needsWhitelist =- request.location?.country === "us-whitelist" ||- request.scrapeOptions?.location?.country === "us-whitelist";-- if (needsWhitelist && !flags?.ipWhitelist) {- return {- error: `Static IP addresses are not enabled for your team. Contact ${SUPPORT_EMAIL} to get a dedicated set of IP addresses you can whitelist.`,- };+ const ipWhitelistError = ipWhitelistRefusal(+ flags,+ request.location?.country,+ request.scrapeOptions?.location?.country,+ );+ if (ipWhitelistError) {+ return { error: ipWhitelistError };}return {};