Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): check the static IP entitlement on search, extract and v0 (#5649)

tomsideguide4ce1241b0686
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 3117651b8..2d225b3a7 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -50,6 +50,7 @@ import {
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { requestCreditsShards } from "../../lib/request-credits-store";
import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
+import { ipWhitelistRefusal } from "../../lib/permissions";
export async function crawlController(req: Request, res: Response) {
try {
@@ -228,6 +229,14 @@ export async function crawlController(req: Request, res: Response) {
});
}
+ const ipWhitelistError = ipWhitelistRefusal(
+ auth.chunk?.flags,
+ pageOptions?.geolocation?.country,
+ );
+ if (ipWhitelistError) {
+ return res.status(403).json({ error: ipWhitelistError });
+ }
+
// if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?
// try {
// const a = new WebScraperDataProvider();
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index e8941e40a..47ac8295e 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -40,6 +40,7 @@ import {
} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
+import { ipWhitelistRefusal } from "../../lib/permissions";
async function scrapeHelper(
jobId: string,
@@ -103,6 +104,18 @@ async function scrapeHelper(
};
}
+ const ipWhitelistError = ipWhitelistRefusal(
+ acuc?.flags,
+ pageOptions?.geolocation?.country,
+ );
+ if (ipWhitelistError) {
+ return {
+ success: false,
+ error: ipWhitelistError,
+ returnCode: 403,
+ };
+ }
+
const { scrapeOptions, internalOptions } = fromV0Combo(
pageOptions,
extractorOptions,
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 042038c7c..f759a0199 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -33,6 +33,7 @@ import {
} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { isUsOnly } from "../../lib/us-only";
+import { ipWhitelistRefusal } from "../../lib/permissions";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
async function searchHelper(
@@ -275,6 +276,14 @@ export async function searchController(req: Request, res: Response) {
};
const origin = req.body.origin ?? "api";
+ const ipWhitelistError =
+ pageOptions.fetchPageContent !== false
+ ? ipWhitelistRefusal(chunk?.flags, pageOptions?.geolocation?.country)
+ : undefined;
+ if (ipWhitelistError) {
+ return res.status(403).json({ error: ipWhitelistError });
+ }
+
const searchOptions = req.body.searchOptions ?? { limit: 5 };
try {
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 77906b59c..b3764ca31 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -36,6 +36,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
+import { ipWhitelistRefusal } from "../../lib/permissions";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
import { isSdkOrigin } from "../../lib/sdk-origin";
async function oldExtract(
@@ -141,6 +142,18 @@ export async function extractController(
});
}
+ const ipWhitelistError = ipWhitelistRefusal(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.location?.country,
+ req.body.scrapeOptions?.geolocation?.country,
+ );
+ if (ipWhitelistError) {
+ return res.status(403).json({
+ success: false,
+ error: ipWhitelistError,
+ });
+ }
+
const invalidURLs: string[] =
req.body.urls?.filter((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 750dfda77..40c258996 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -28,7 +28,7 @@ import {
resolveSafeMode,
getEffectiveSearchForcedKind,
} from "../../lib/safe-mode";
-import { checkPermissions } from "../../lib/permissions";
+import { checkPermissions, ipWhitelistRefusal } from "../../lib/permissions";
import {
DocumentWithCostTracking,
scrapeSearchResults,
@@ -216,6 +216,21 @@ export async function searchController(
});
}
+ const ipWhitelistError =
+ requestedFormats.length > 0
+ ? ipWhitelistRefusal(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.location?.country,
+ req.body.scrapeOptions?.geolocation?.country,
+ )
+ : undefined;
+ if (ipWhitelistError) {
+ return res.status(403).json({
+ success: false,
+ error: ipWhitelistError,
+ });
+ }
+
// Search only scrapes (and only honors scrapeOptions) when formats are
// requested, so the scrape-option checks apply only then.
if (
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index 6f37af49b..530e99e09 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -24,6 +24,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
+import { ipWhitelistRefusal } from "../../lib/permissions";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
import { checkExtractAvailable } from "../../lib/extract/fire-0/extraction-service-f0";
@@ -60,6 +61,17 @@ export async function extractController(
});
}
+ const ipWhitelistError = ipWhitelistRefusal(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.location?.country,
+ );
+ if (ipWhitelistError) {
+ return res.status(403).json({
+ success: false,
+ error: ipWhitelistError,
+ });
+ }
+
const extractId = uuidv7();
const createdAt = Date.now();
_logger.info("Extract starting...", {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index ffe06a9be..6165998f7 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -47,7 +47,7 @@ import {
isLockdownZeroDataRetention,
getEffectiveSearchForcedKind,
} from "../../lib/safe-mode";
-import { checkPermissions } from "../../lib/permissions";
+import { checkPermissions, ipWhitelistRefusal } from "../../lib/permissions";
import {
actionTypesOf,
checkKeyEndpointRestriction,
@@ -334,6 +334,20 @@ async function searchControllerInner(
});
}
+ const ipWhitelistError =
+ requestedFormats.length > 0
+ ? ipWhitelistRefusal(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.location?.country,
+ )
+ : undefined;
+ if (ipWhitelistError) {
+ return res.status(403).json({
+ success: false,
+ error: ipWhitelistError,
+ });
+ }
+
// Search only scrapes (and only honors scrapeOptions) when formats are
// requested, so the scrape-option checks apply only then.
if (
diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.ts
index 6eb1afd52..8ba1ee832 100644
--- a/apps/api/src/lib/permissions.ts
+++ b/apps/api/src/lib/permissions.ts
@@ -55,6 +55,20 @@ interface PermissionOptions {
const SUPPORT_EMAIL = "[email protected]";
+/** The refusal for static-IP egress ("us-whitelist") without the entitlement. */
+export function ipWhitelistRefusal(
+ flags: TeamFlags | null | undefined,
+ ...countries: unknown[]
+): string | undefined {
+ const needsWhitelist = countries.some(
+ value =>
+ typeof value === "string" && value.toLowerCase() === "us-whitelist",
+ );
+ return needsWhitelist && !flags?.ipWhitelist
+ ? `Static IP addresses are not enabled for your team. Contact ${SUPPORT_EMAIL} to get a dedicated set of IP addresses you can whitelist.`
+ : undefined;
+}
+
export function checkPermissions(
request: APIRequest,
flags?: TeamFlags,
@@ -185,15 +199,13 @@ export function checkPermissions(
return { error: usOnlyLocationError, code: "US_ONLY_UNAVAILABLE" };
}
- // ip whitelist perms
- const needsWhitelist =
- request.location?.country === "us-whitelist" ||
- request.scrapeOptions?.location?.country === "us-whitelist";
-
- if (needsWhitelist && !flags?.ipWhitelist) {
- return {
- error: `Static IP addresses are not enabled for your team. Contact ${SUPPORT_EMAIL} to get a dedicated set of IP addresses you can whitelist.`,
- };
+ const ipWhitelistError = ipWhitelistRefusal(
+ flags,
+ request.location?.country,
+ request.scrapeOptions?.location?.country,
+ );
+ if (ipWhitelistError) {
+ return { error: ipWhitelistError };
}
return {};