Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Improve URL filtering error messages with specific denial reasons (FIR-2352) (#1676)
devin-ai-integration[bot]9710bdffc050
diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.tsindex 897ea46c0..7f2598e15 100644--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts@@ -70,7 +70,7 @@ describe("WebCrawler", () => {10,);- expect(filteredLinks.length).toBe(limit); // Check if the number of results respects the limit- expect(filteredLinks).toEqual([initialUrl, initialUrl + "/page1"]);+ expect(filteredLinks.links.length).toBe(limit); // Check if the number of results respects the limit+ expect(filteredLinks.links).toEqual([initialUrl, initialUrl + "/page1"]);});});diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 899e51574..91a54b315 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -10,6 +10,30 @@ import https from "https";import { redisEvictConnection } from "../../services/redis";import { extractLinks } from "../../lib/html-transformer";import { TimeoutSignal } from "../../controllers/v1/types";++export interface FilterResult {+ allowed: boolean;+ url?: string;+ denialReason?: string;+}++export enum DenialReason {+ DEPTH_LIMIT = "URL exceeds maximum crawl depth",+ EXCLUDE_PATTERN = "URL matches exclude pattern",+ INCLUDE_PATTERN = "URL does not match required include pattern",+ ROBOTS_TXT = "URL blocked by robots.txt",+ FILE_TYPE = "URL points to a file type that is not crawled",+ URL_PARSE_ERROR = "URL could not be parsed",+ BACKWARD_CRAWLING = "URL not allowed due to backward crawling restrictions",+ SOCIAL_MEDIA = "URL is a social media or email link",+ EXTERNAL_LINK = "External URL not allowed",+ SECTION_LINK = "URL contains section anchor (#)"+}++export interface FilterLinksResult {+ links: string[];+ denialReasons: Map<string, string>;+}export class WebCrawler {private jobId: string;private initialUrl: string;@@ -97,18 +121,23 @@ export class WebCrawler {limit: number,maxDepth: number,fromMap: boolean = false,- ): string[] {+ ): FilterLinksResult {+ const denialReasons = new Map<string, string>();+if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {this.logger.debug("Max discovery depth hit, filtering off all links", { currentDiscoveryDepth: this.currentDiscoveryDepth, maxDiscoveryDepth: this.maxDiscoveryDepth });- return [];+ sitemapLinks.forEach(link => {+ denialReasons.set(link, "Maximum discovery depth reached");+ });+ return { links: [], denialReasons };}// If the initial URL is a sitemap.xml, skip filteringif (this.initialUrl.endsWith("sitemap.xml") && fromMap) {- return sitemapLinks.slice(0, limit);+ return { links: sitemapLinks.slice(0, limit), denialReasons };}- return sitemapLinks+ const filteredLinks = sitemapLinks.filter((link) => {let url: URL;try {@@ -130,6 +159,7 @@ export class WebCrawler {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} DEPTH FAIL`);}+ denialReasons.set(link, DenialReason.DEPTH_LIMIT);return false;}@@ -145,6 +175,7 @@ export class WebCrawler {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} EXCLUDE FAIL`);}+ denialReasons.set(link, DenialReason.EXCLUDE_PATTERN);return false;}}@@ -159,6 +190,7 @@ export class WebCrawler {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} INCLUDE FAIL`);}+ denialReasons.set(link, DenialReason.INCLUDE_PATTERN);return false;}}@@ -193,6 +225,7 @@ export class WebCrawler {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`);}+ denialReasons.set(link, DenialReason.BACKWARD_CRAWLING);return false;}}@@ -209,6 +242,7 @@ export class WebCrawler {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} ROBOTS FAIL`);}+ denialReasons.set(link, DenialReason.ROBOTS_TXT);return false;}@@ -216,6 +250,7 @@ export class WebCrawler {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} FILE FAIL`);}+ denialReasons.set(link, DenialReason.FILE_TYPE);return false;}@@ -225,6 +260,8 @@ export class WebCrawler {return true;}).slice(0, limit);++ return { links: filteredLinks, denialReasons };}public async getRobotsTxt(skipTlsVerification = false, abort?: AbortSignal): Promise<string> {@@ -277,12 +314,13 @@ export class WebCrawler {if (fromMap && onlySitemap) {return await urlsHandler(urls);} else {- let filteredLinks = this.filterLinks(- [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl) !== null),+ let filteredLinksResult = this.filterLinks(+ [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl).allowed),leftOfLimit,this.maxCrawledDepth,fromMap,);+ let filteredLinks = filteredLinksResult.links;leftOfLimit -= filteredLinks.length;let uniqueURLs: string[] = [];for (const url of filteredLinks) {@@ -357,38 +395,34 @@ export class WebCrawler {}}- public filterURL(href: string, url: string): string | null {+ public filterURL(href: string, url: string): FilterResult {let fullUrl = href;if (!href.startsWith("http")) {try {fullUrl = new URL(href, url).toString();} catch (_) {- return null;+ return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };}}let urlObj;try {urlObj = new URL(fullUrl);} catch (_) {- return null;+ return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };}const path = urlObj.pathname;if (this.isInternalLink(fullUrl)) {// INTERNAL LINKS- if (- this.isInternalLink(fullUrl) &&- this.noSections(fullUrl) &&- !this.matchesExcludes(path) &&- this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)- ) {- return fullUrl;- } else if (- this.isInternalLink(fullUrl) &&- this.noSections(fullUrl) &&- !this.matchesExcludes(path) &&- !this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)- ) {+ if (!this.noSections(fullUrl)) {+ return { allowed: false, denialReason: DenialReason.SECTION_LINK };+ }++ if (this.matchesExcludes(path)) {+ return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };+ }++ if (!this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)) {(async () => {await redisEvictConnection.sadd("crawl:" + this.jobId + ":robots_blocked",@@ -399,18 +433,29 @@ export class WebCrawler {24 * 60 * 60,);})();+ return { allowed: false, denialReason: DenialReason.ROBOTS_TXT };}++ return { allowed: true, url: fullUrl };} else {// EXTERNAL LINKS+ if (this.isSocialMediaOrEmail(fullUrl)) {+ return { allowed: false, denialReason: DenialReason.SOCIAL_MEDIA };+ }++ if (this.matchesExcludes(fullUrl, true)) {+ return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };+ }+if (this.isInternalLink(url) &&this.allowExternalContentLinks &&- !this.isSocialMediaOrEmail(fullUrl) &&- !this.matchesExcludes(fullUrl, true) &&!this.isExternalMainPage(fullUrl)) {- return fullUrl;+ return { allowed: true, url: fullUrl };}++ return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };}if (@@ -418,14 +463,22 @@ export class WebCrawler {!this.isSocialMediaOrEmail(fullUrl) &&this.isSubdomain(fullUrl)) {- return fullUrl;+ return { allowed: true, url: fullUrl };}- return null;+ return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };}private async extractLinksFromHTMLRust(html: string, url: string) {- return (await extractLinks(html)).filter(x => this.filterURL(x, url));+ const links = await extractLinks(html);+ const filteredLinks: string[] = [];+ for (const link of links) {+ const filterResult = this.filterURL(link, url);+ if (filterResult.allowed && filterResult.url) {+ filteredLinks.push(filterResult.url);+ }+ }+ return filteredLinks;}private extractLinksFromHTMLCheerio(html: string, url: string) {@@ -438,9 +491,9 @@ export class WebCrawler {if (href.match(/^https?:\/[^\/]/)) {href = href.replace(/^https?:\//, "$&/");}- const u = this.filterURL(href, url);- if (u !== null) {- links.push(u);+ const filterResult = this.filterURL(href, url);+ if (filterResult.allowed && filterResult.url) {+ links.push(filterResult.url);}}});diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 7ff5283fc..deff1f0c8 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -196,13 +196,13 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {: sc.crawlerOptions.limit -(await getDoneJobsOrderedLength(job.data.crawl_id));- if (univistedUrls.length !== 0 && addableJobCount > 0) {+ if (univistedUrls.links.length !== 0 && addableJobCount > 0) {logger.info("Adding jobs", {- univistedUrls: univistedUrls.length,+ univistedUrls: univistedUrls.links.length,addableJobCount,});- const jobs = univistedUrls.slice(0, addableJobCount).map((url) => {+ const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {const uuid = uuidv4();return {name: uuid,@@ -949,12 +949,13 @@ async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: ssc.crawlerOptions.limit ?? 10000,);- const validIndexLinks = crawler.filterLinks(- index.filter(x => crawler.filterURL(x, trimmedURL.href) !== null),+ const validIndexLinksResult = crawler.filterLinks(+ index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),sc.crawlerOptions.limit ?? 10000,sc.crawlerOptions.maxDepth ?? 10,false,);+ const validIndexLinks = validIndexLinksResult.links;return validIndexLinks;}@@ -1308,14 +1309,10 @@ async function processJob(job: Job & { id: string }, token: string) {job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape) {const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);- if (- crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL) ===- null &&- !job.data.isCrawlSourceScrape- ) {- throw new Error(- "Redirected target URL is not allowed by crawlOptions",- ); // TODO: make this its own error type that is ignored by error tracking+ const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);+ if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {+ const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";+ throw new Error(reason);}// Only re-set originUrl if it's different from the current hostname@@ -1379,11 +1376,11 @@ async function processJob(job: Job & { id: string }, token: string) {Infinity,sc.crawlerOptions?.maxDepth ?? 10,);- logger.debug("Discovered " + links.length + " links...", {- linksLength: links.length,+ logger.debug("Discovered " + links.links.length + " links...", {+ linksLength: links.links.length,});- for (const link of links) {+ for (const link of links.links) {if (await lockURL(job.data.crawl_id, sc, link)) {// This seems to work really welelconst jobPriority = await getJobPriority({@@ -1442,17 +1439,17 @@ async function processJob(job: Job & { id: string }, token: string) {}// Only run check after adding new jobs for discovery - mogery- if (- job.data.isCrawlSourceScrape &&- crawler.filterLinks(+ if (job.data.isCrawlSourceScrape) {+ const filterResult = crawler.filterLinks([doc.metadata.url ?? doc.metadata.sourceURL!],1,sc.crawlerOptions?.maxDepth ?? 10,- ).length === 0- ) {- throw new Error(- "Source URL is not allowed by includePaths/excludePaths rules",);+ if (filterResult.links.length === 0) {+ const url = doc.metadata.url ?? doc.metadata.sourceURL!;+ const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";+ throw new Error(reason);+ }}}}@@ -1701,4 +1698,4 @@ app.listen(3005, () => {console.log("All jobs finished. Worker out!");process.exit(0);-})();\ No newline at end of file+})();