Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add static IP proxy pool (#2103)
tomde246c5e827b
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex ecb7cbaed..9cbd93716 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -23,30 +23,30 @@ import { logger as _logger } from "../../lib/logger";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";+import { checkPermissions } from "../../lib/permissions";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,res: Response<BatchScrapeResponse>,) {const preNormalizedBody = { ...req.body };+ if (req.body?.ignoreInvalidURLs === true) {+ req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);+ } else {+ req.body = batchScrapeRequestSchema.parse(req.body);+ }- if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {- return res.status(400).json({+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({success: false,- error:- "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ error: permissions.error,});}const zeroDataRetention =req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;- if (req.body?.ignoreInvalidURLs === true) {- req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);- } else {- req.body = batchScrapeRequestSchema.parse(req.body);- }-const id = req.body.appendToId ?? uuidv4();const logger = _logger.child({crawlId: id,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 39025f117..3e4399927 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -11,6 +11,7 @@ import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";+import { checkPermissions } from "../../lib/permissions";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -19,11 +20,11 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);- if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {- return res.status(400).json({+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({success: false,- error:- "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ error: permissions.error,});}diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 45593c2ca..4ba61a3c1 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -28,6 +28,7 @@ import {queryIndexAtSplitLevel,} from "../../services/index";import { MapTimeoutError } from "../../lib/error";+import { checkPermissions } from "../../lib/permissions";configDotenv();const redis = new Redis(process.env.REDIS_URL!);@@ -360,6 +361,14 @@ export async function mapController(});}+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({+ success: false,+ error: permissions.error,+ });+ }+logger.info("Map request", {request: req.body,originalRequest,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex d008bc002..a8a85447e 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -13,6 +13,7 @@ import { getJobPriority } from "../../lib/job-priority";import { getScrapeQueue } from "../../services/queue-service";import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";+import { checkPermissions } from "../../lib/permissions";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -20,12 +21,13 @@ export async function scrapeController() {const jobId: string = uuidv4();const preNormalizedBody = { ...req.body };+ req.body = scrapeRequestSchema.parse(req.body);- if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {- return res.status(400).json({+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({success: false,- error:- "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ error: permissions.error,});}@@ -48,8 +50,6 @@ export async function scrapeController(account: req.account,});- req.body = scrapeRequestSchema.parse(req.body);-const origin = req.body.origin;const timeout = req.body.timeout;diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 3b2abdaf0..0c3cac160 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -270,6 +270,8 @@ export const actionsSchema = z},);+const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];+export const locationSchema = z.object({country: z@@ -279,14 +281,14 @@ export const locationSchema = zval =>!val ||Object.keys(countries).includes(val.toUpperCase()) ||- val === "US-generic",- {- message:- "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",- },+ SPECIAL_COUNTRIES.includes(val.toLowerCase()),+ "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",)- .transform(val => (val ? val.toUpperCase() : "US-generic")),- languages: z.string().array().optional(),+ .transform(val => {+ if (!val) return "us-generic";+ return val.toLowerCase();+ }),+ languages: z.array(z.string()).optional(),}).optional();@@ -1115,6 +1117,7 @@ export type TeamFlags = {checkRobotsOnScrape?: boolean;allowTeammateInvites?: boolean;crawlTtlHours?: number;+ ipWhitelist?: boolean;} | null;export type AuthCreditUsageChunkFromTeam = Omit<diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex c00d9a0e7..fe17023de 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -23,30 +23,30 @@ import { createWebhookSender, WebhookEvent } from "../../services/webhook";import { logger as _logger } from "../../lib/logger";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { checkPermissions } from "../../lib/permissions";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,res: Response<BatchScrapeResponse>,) {const preNormalizedBody = { ...req.body };+ if (req.body?.ignoreInvalidURLs === true) {+ req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);+ } else {+ req.body = batchScrapeRequestSchema.parse(req.body);+ }- if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {- return res.status(400).json({+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({success: false,- error:- "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ error: permissions.error,});}const zeroDataRetention =req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;- if (req.body?.ignoreInvalidURLs === true) {- req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);- } else {- req.body = batchScrapeRequestSchema.parse(req.body);- }-const id = req.body.appendToId ?? uuidv4();const logger = _logger.child({crawlId: id,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 337e6f806..ec0ff91b8 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -12,6 +12,7 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";+import { checkPermissions } from "../../lib/permissions";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -20,11 +21,11 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);- if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {- return res.status(400).json({+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({success: false,- error:- "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ error: permissions.error,});}diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 7df9e6fcf..849250c5c 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -30,6 +30,7 @@ import {import { redisEvictConnection } from "../../services/redis";import { performCosineSimilarityV2 } from "../../lib/map-cosine";import { MapTimeoutError } from "../../lib/error";+import { checkPermissions } from "../../lib/permissions";configDotenv();@@ -354,6 +355,14 @@ export async function mapController(const originalRequest = req.body;req.body = mapRequestSchema.parse(req.body);+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({+ success: false,+ error: permissions.error,+ });+ }+logger.info("Map request", {request: req.body,originalRequest,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex af1017fd8..7200f0865 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -13,6 +13,7 @@ import { getJobPriority } from "../../lib/job-priority";import { getScrapeQueue } from "../../services/queue-service";import { hasFormatOfType } from "../../lib/format-utils";import { TransportableError } from "../../lib/error";+import { checkPermissions } from "../../lib/permissions";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -20,12 +21,13 @@ export async function scrapeController() {const jobId = uuidv4();const preNormalizedBody = { ...req.body };+ req.body = scrapeRequestSchema.parse(req.body);- if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {- return res.status(400).json({+ const permissions = checkPermissions(req.body, req.acuc?.flags);+ if (permissions.error) {+ return res.status(403).json({success: false,- error:- "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ error: permissions.error,});}@@ -48,8 +50,6 @@ export async function scrapeController(account: req.account,});- req.body = scrapeRequestSchema.parse(req.body);-const origin = req.body.origin;const timeout = req.body.timeout;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 89ad8cfbd..12fc05b8f 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -305,7 +305,9 @@ function transformIframeSelector(selector: string): string {});}-const locationSchema = z+const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];++export const locationSchema = z.object({country: z.string()@@ -314,14 +316,14 @@ const locationSchema = zval =>!val ||Object.keys(countries).includes(val.toUpperCase()) ||- val === "US-generic",- {- message:- "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",- },+ SPECIAL_COUNTRIES.includes(val.toLowerCase()),+ "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",)- .transform(val => (val ? val.toUpperCase() : "US-generic")),- languages: z.string().array().optional(),+ .transform(val => {+ if (!val) return "us-generic";+ return val.toLowerCase();+ }),+ languages: z.array(z.string()).optional(),}).optional();@@ -959,6 +961,7 @@ export type TeamFlags = {checkRobotsOnScrape?: boolean;allowTeammateInvites?: boolean;crawlTtlHours?: number;+ ipWhitelist?: boolean;} | null;export interface RequestWithMaybeACUC<diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsnew file mode 100644index 000000000..3a0d6a5c1--- /dev/null+++ b/apps/api/src/lib/permissions.ts@@ -0,0 +1,38 @@+import { TeamFlags } from "../controllers/v2/types";++type LocationOptions = { country?: string };++interface APIRequest {+ zeroDataRetention?: boolean;+ location?: LocationOptions;+ scrapeOptions?: {+ location?: LocationOptions;+ };+}++const SUPPORT_EMAIL = "[email protected]";++export function checkPermissions(+ request: APIRequest,+ flags?: TeamFlags,+): { error?: string } {+ // zdr perms+ if (request.zeroDataRetention && !flags?.allowZDR) {+ return {+ error: `Zero Data Retention (ZDR) is not enabled for your team. Contact ${SUPPORT_EMAIL} to enable this feature.`,+ };+ }++ // ip whitelist perms+ const needsWhitelist =+ request.location?.country === "us-whitelist" ||+ request.scrapeOptions?.location?.country === "us-whitelist";++ if (needsWhitelist && !flags?.ipWhitelist) {+ return {+ error: `Static IP addresses are not enabled for your team. Contact ${SUPPORT_EMAIL} to get a dedicated set of IP addresses you can whitelist.`,+ };+ }++ return {};+}