Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): re-check blocklist on post-redirect URL for all scrape job types (#4581)
Gergő Móricz58d0c8ebb618
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex f5d853f26..5cec2e7ae 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -46,7 +46,7 @@ import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";-import { CrawlDenialError } from "../../lib/error";+import { UnsupportedSiteError } from "../../lib/error";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -159,7 +159,7 @@ export async function batchScrapeController(apiKeyId: req.acuc?.api_key_id ?? null,auditMetadata: req.body.auditMetadata,url,- error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ error: new UnsupportedSiteError(),origin: req.body.origin ?? "api",integration: req.body.integration,zeroDataRetention: zeroDataRetention ?? false,@@ -184,7 +184,7 @@ export async function batchScrapeController(apiKeyId: req.acuc?.api_key_id ?? null,auditMetadata: req.body.auditMetadata,url,- error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ error: new UnsupportedSiteError(),origin: req.body.origin ?? "api",integration: req.body.integration,zeroDataRetention: zeroDataRetention ?? false,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 487c7ccc0..61d10d50c 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -32,7 +32,7 @@ import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";-import { CrawlDenialError } from "../../lib/error";+import { UnsupportedSiteError } from "../../lib/error";async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,@@ -142,7 +142,7 @@ export async function extractController(apiKeyId: req.acuc?.api_key_id ?? null,auditMetadata: req.body.scrapeOptions?.auditMetadata,url,- error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ error: new UnsupportedSiteError(),origin: req.body.origin ?? "api",integration: req.body.integration,zeroDataRetention: false,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 279ac1dd3..9a43d33dd 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -335,6 +335,14 @@ async function scrapeControllerInner(});}+ if (e.code === "UNSUPPORTED_SITE") {+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+if (e.code === "SCRAPE_MEDIA_ACCESS_DENIED") {return res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 0af5b7729..4c20661c3 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -48,7 +48,7 @@ import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";-import { CrawlDenialError } from "../../lib/error";+import { UnsupportedSiteError } from "../../lib/error";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -181,7 +181,7 @@ export async function batchScrapeController(apiKeyId: req.acuc?.api_key_id ?? null,auditMetadata: req.body.auditMetadata,url,- error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ error: new UnsupportedSiteError(),origin: req.body.origin ?? "api",integration: req.body.integration,zeroDataRetention,@@ -206,7 +206,7 @@ export async function batchScrapeController(apiKeyId: req.acuc?.api_key_id ?? null,auditMetadata: req.body.auditMetadata,url,- error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ error: new UnsupportedSiteError(),origin: req.body.origin ?? "api",integration: req.body.integration,zeroDataRetention,diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 6bdab11a2..a83f59580 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -23,7 +23,7 @@ import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";-import { CrawlDenialError } from "../../lib/error";+import { UnsupportedSiteError } from "../../lib/error";/*** Extracts data from the provided URLs based on the request parameters.@@ -84,7 +84,7 @@ export async function extractController(apiKeyId: req.acuc?.api_key_id ?? null,auditMetadata: req.body.scrapeOptions?.auditMetadata,url,- error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ error: new UnsupportedSiteError(),origin: req.body.origin ?? "api",integration: req.body.integration,zeroDataRetention: false,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 9615dba9f..da7e24665 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -470,6 +470,17 @@ export async function scrapeController(});}+ if (e.code === "UNSUPPORTED_SITE") {+ setSpanAttributes(span, {+ "scrape.status_code": 403,+ });+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+if (e.code === "SCRAPE_MEDIA_ACCESS_DENIED") {setSpanAttributes(span, {"scrape.status_code": 403,diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 8348af298..7e2fb120f 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -10,6 +10,7 @@ import {SitemapError,TransportableError,UnknownError,+ UnsupportedSiteError,} from "./error";import {ActionError,@@ -76,6 +77,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,+ UNSUPPORTED_SITE: UnsupportedSiteError,SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,SCRAPE_VIDEO_UNSUPPORTED_URL: VideoUnsupportedUrlError,SCRAPE_MEDIA_ACCESS_DENIED: MediaAccessDeniedError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 39cd45096..a694ed141 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -1,3 +1,5 @@+import { UNSUPPORTED_SITE_MESSAGE } from "./strings";+export type ErrorCodes =| "THIRD_PARTY_DATA_TERMS_REQUIRED"| "SCRAPE_TIMEOUT"@@ -36,6 +38,7 @@ export type ErrorCodes =| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"| "PARSE_UNSUPPORTED_OPTIONS"| "CRAWL_DENIAL"+ | "UNSUPPORTED_SITE"| "MAP_FAILED"| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST"@@ -368,6 +371,25 @@ export class CrawlDenialError extends TransportableError {}}+export class UnsupportedSiteError extends TransportableError {+ constructor() {+ super("UNSUPPORTED_SITE", UNSUPPORTED_SITE_MESSAGE);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new UnsupportedSiteError();+ x.stack = data.stack;+ return x;+ }+}+export class ActionsNotSupportedError extends TransportableError {constructor(message: string) {super("SCRAPE_ACTIONS_NOT_SUPPORTED", message);diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.tsindex e87f78800..1416105fa 100644--- a/apps/api/src/lib/siem-logging/event.ts+++ b/apps/api/src/lib/siem-logging/event.ts@@ -5,6 +5,7 @@ import {CrawlDenialError,JobCancelledError,TransportableError,+ UnsupportedSiteError,} from "../error";import { UnsafeDomainBlockedError } from "../threat-protection/error";import type {@@ -69,6 +70,7 @@ function resultForOutcome(if (outcome.success) return "success";if (outcome.error instanceof CrawlDenialError ||+ outcome.error instanceof UnsupportedSiteError ||outcome.error instanceof UnsafeDomainBlockedError ||outcome.threatDecisions?.some(decision => !decision.allowed)) {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e8b3737c1..5ebe2ad73 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -58,7 +58,7 @@ import { CostTracking } from "../../lib/cost-tracking";import { chargeKeylessCredits } from "../../lib/keyless";import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";+import { generateURLSplits, queryIndexAtSplitLevel } from "../index";import { WebCrawler } from "../../scraper/WebScraper/crawler";import {@@ -77,8 +77,10 @@ import {SitemapError,TransportableError,UnknownError,+ UnsupportedSiteError,} from "../../lib/error";import { serializeTransportableError } from "../../lib/error-serde";+import { canonicalizeUrl } from "../../lib/threat-protection/providers/web-risk/canonicalize";import { trackScrape } from "../../lib/tracking";import type { NuQJob } from "./nuq";import {@@ -433,6 +435,33 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const doc = pipeline.document;+ if (+ pipeline.exchange === undefined &&+ job.data.origin !== "monitor" &&+ !job.data.internalOptions?.isParse &&+ doc.metadata.url !== undefined &&+ doc.metadata.sourceURL !== undefined &&+ canonicalizeUrl(doc.metadata.url) !==+ canonicalizeUrl(doc.metadata.sourceURL)+ ) {+ let teamFlags = job.data.internalOptions?.teamFlags ?? null;+ let orgId = job.data.internalOptions?.orgId ?? null;+ if (job.data.internalOptions?.teamFlags === undefined) {+ const teamChunk = await getACUCTeam(job.data.team_id);+ teamFlags = teamChunk?.flags ?? null;+ orgId = orgId ?? teamChunk?.org_id ?? null;+ }+ if (+ isUrlBlocked(doc.metadata.url, teamFlags, {+ team_id: job.data.team_id,+ org_id: orgId,+ origin: job.data.origin,+ })+ ) {+ throw new UnsupportedSiteError();+ }+ }+const rawHtml = doc.rawHtml ?? "";if (!hasFormatOfType(job.data.scrapeOptions.formats, "rawHtml")) {@@ -530,17 +559,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {await saveCrawl(job.data.crawl_id, sc);}- const teamChunk = await getACUCTeam(job.data.team_id);- if (- isUrlBlocked(doc.metadata.url, teamChunk?.flags ?? null, {- team_id: job.data.team_id,- org_id: teamChunk?.org_id ?? null,- origin: job.data.origin,- })- ) {- throw new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE); // TODO: make this its own error type that is ignored by error tracking- }-const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));const p2 = generateURLPermutations(normalizeURL(doc.metadata.sourceURL, sc),