Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Improve URL filtering error messages with specific denial reasons (FIR-2352) (#1676)

devin-ai-integration[bot]9710bdffc050
diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
index 897ea46c0..7f2598e15 100644
--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
@@ -70,7 +70,7 @@ describe("WebCrawler", () => {
10,
);
- expect(filteredLinks.length).toBe(limit); // Check if the number of results respects the limit
- expect(filteredLinks).toEqual([initialUrl, initialUrl + "/page1"]);
+ expect(filteredLinks.links.length).toBe(limit); // Check if the number of results respects the limit
+ expect(filteredLinks.links).toEqual([initialUrl, initialUrl + "/page1"]);
});
});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 899e51574..91a54b315 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -10,6 +10,30 @@ import https from "https";
import { redisEvictConnection } from "../../services/redis";
import { extractLinks } from "../../lib/html-transformer";
import { TimeoutSignal } from "../../controllers/v1/types";
+
+export interface FilterResult {
+ allowed: boolean;
+ url?: string;
+ denialReason?: string;
+}
+
+export enum DenialReason {
+ DEPTH_LIMIT = "URL exceeds maximum crawl depth",
+ EXCLUDE_PATTERN = "URL matches exclude pattern",
+ INCLUDE_PATTERN = "URL does not match required include pattern",
+ ROBOTS_TXT = "URL blocked by robots.txt",
+ FILE_TYPE = "URL points to a file type that is not crawled",
+ URL_PARSE_ERROR = "URL could not be parsed",
+ BACKWARD_CRAWLING = "URL not allowed due to backward crawling restrictions",
+ SOCIAL_MEDIA = "URL is a social media or email link",
+ EXTERNAL_LINK = "External URL not allowed",
+ SECTION_LINK = "URL contains section anchor (#)"
+}
+
+export interface FilterLinksResult {
+ links: string[];
+ denialReasons: Map<string, string>;
+}
export class WebCrawler {
private jobId: string;
private initialUrl: string;
@@ -97,18 +121,23 @@ export class WebCrawler {
limit: number,
maxDepth: number,
fromMap: boolean = false,
- ): string[] {
+ ): FilterLinksResult {
+ const denialReasons = new Map<string, string>();
+
if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {
this.logger.debug("Max discovery depth hit, filtering off all links", { currentDiscoveryDepth: this.currentDiscoveryDepth, maxDiscoveryDepth: this.maxDiscoveryDepth });
- return [];
+ sitemapLinks.forEach(link => {
+ denialReasons.set(link, "Maximum discovery depth reached");
+ });
+ return { links: [], denialReasons };
}
// If the initial URL is a sitemap.xml, skip filtering
if (this.initialUrl.endsWith("sitemap.xml") && fromMap) {
- return sitemapLinks.slice(0, limit);
+ return { links: sitemapLinks.slice(0, limit), denialReasons };
}
- return sitemapLinks
+ const filteredLinks = sitemapLinks
.filter((link) => {
let url: URL;
try {
@@ -130,6 +159,7 @@ export class WebCrawler {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} DEPTH FAIL`);
}
+ denialReasons.set(link, DenialReason.DEPTH_LIMIT);
return false;
}
@@ -145,6 +175,7 @@ export class WebCrawler {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} EXCLUDE FAIL`);
}
+ denialReasons.set(link, DenialReason.EXCLUDE_PATTERN);
return false;
}
}
@@ -159,6 +190,7 @@ export class WebCrawler {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} INCLUDE FAIL`);
}
+ denialReasons.set(link, DenialReason.INCLUDE_PATTERN);
return false;
}
}
@@ -193,6 +225,7 @@ export class WebCrawler {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`);
}
+ denialReasons.set(link, DenialReason.BACKWARD_CRAWLING);
return false;
}
}
@@ -209,6 +242,7 @@ export class WebCrawler {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} ROBOTS FAIL`);
}
+ denialReasons.set(link, DenialReason.ROBOTS_TXT);
return false;
}
@@ -216,6 +250,7 @@ export class WebCrawler {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} FILE FAIL`);
}
+ denialReasons.set(link, DenialReason.FILE_TYPE);
return false;
}
@@ -225,6 +260,8 @@ export class WebCrawler {
return true;
})
.slice(0, limit);
+
+ return { links: filteredLinks, denialReasons };
}
public async getRobotsTxt(skipTlsVerification = false, abort?: AbortSignal): Promise<string> {
@@ -277,12 +314,13 @@ export class WebCrawler {
if (fromMap && onlySitemap) {
return await urlsHandler(urls);
} else {
- let filteredLinks = this.filterLinks(
- [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl) !== null),
+ let filteredLinksResult = this.filterLinks(
+ [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl).allowed),
leftOfLimit,
this.maxCrawledDepth,
fromMap,
);
+ let filteredLinks = filteredLinksResult.links;
leftOfLimit -= filteredLinks.length;
let uniqueURLs: string[] = [];
for (const url of filteredLinks) {
@@ -357,38 +395,34 @@ export class WebCrawler {
}
}
- public filterURL(href: string, url: string): string | null {
+ public filterURL(href: string, url: string): FilterResult {
let fullUrl = href;
if (!href.startsWith("http")) {
try {
fullUrl = new URL(href, url).toString();
} catch (_) {
- return null;
+ return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };
}
}
let urlObj;
try {
urlObj = new URL(fullUrl);
} catch (_) {
- return null;
+ return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };
}
const path = urlObj.pathname;
if (this.isInternalLink(fullUrl)) {
// INTERNAL LINKS
- if (
- this.isInternalLink(fullUrl) &&
- this.noSections(fullUrl) &&
- !this.matchesExcludes(path) &&
- this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)
- ) {
- return fullUrl;
- } else if (
- this.isInternalLink(fullUrl) &&
- this.noSections(fullUrl) &&
- !this.matchesExcludes(path) &&
- !this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)
- ) {
+ if (!this.noSections(fullUrl)) {
+ return { allowed: false, denialReason: DenialReason.SECTION_LINK };
+ }
+
+ if (this.matchesExcludes(path)) {
+ return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };
+ }
+
+ if (!this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)) {
(async () => {
await redisEvictConnection.sadd(
"crawl:" + this.jobId + ":robots_blocked",
@@ -399,18 +433,29 @@ export class WebCrawler {
24 * 60 * 60,
);
})();
+ return { allowed: false, denialReason: DenialReason.ROBOTS_TXT };
}
+
+ return { allowed: true, url: fullUrl };
} else {
// EXTERNAL LINKS
+ if (this.isSocialMediaOrEmail(fullUrl)) {
+ return { allowed: false, denialReason: DenialReason.SOCIAL_MEDIA };
+ }
+
+ if (this.matchesExcludes(fullUrl, true)) {
+ return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };
+ }
+
if (
this.isInternalLink(url) &&
this.allowExternalContentLinks &&
- !this.isSocialMediaOrEmail(fullUrl) &&
- !this.matchesExcludes(fullUrl, true) &&
!this.isExternalMainPage(fullUrl)
) {
- return fullUrl;
+ return { allowed: true, url: fullUrl };
}
+
+ return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };
}
if (
@@ -418,14 +463,22 @@ export class WebCrawler {
!this.isSocialMediaOrEmail(fullUrl) &&
this.isSubdomain(fullUrl)
) {
- return fullUrl;
+ return { allowed: true, url: fullUrl };
}
- return null;
+ return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };
}
private async extractLinksFromHTMLRust(html: string, url: string) {
- return (await extractLinks(html)).filter(x => this.filterURL(x, url));
+ const links = await extractLinks(html);
+ const filteredLinks: string[] = [];
+ for (const link of links) {
+ const filterResult = this.filterURL(link, url);
+ if (filterResult.allowed && filterResult.url) {
+ filteredLinks.push(filterResult.url);
+ }
+ }
+ return filteredLinks;
}
private extractLinksFromHTMLCheerio(html: string, url: string) {
@@ -438,9 +491,9 @@ export class WebCrawler {
if (href.match(/^https?:\/[^\/]/)) {
href = href.replace(/^https?:\//, "$&/");
}
- const u = this.filterURL(href, url);
- if (u !== null) {
- links.push(u);
+ const filterResult = this.filterURL(href, url);
+ if (filterResult.allowed && filterResult.url) {
+ links.push(filterResult.url);
}
}
});
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 7ff5283fc..deff1f0c8 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -196,13 +196,13 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
: sc.crawlerOptions.limit -
(await getDoneJobsOrderedLength(job.data.crawl_id));
- if (univistedUrls.length !== 0 && addableJobCount > 0) {
+ if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
logger.info("Adding jobs", {
- univistedUrls: univistedUrls.length,
+ univistedUrls: univistedUrls.links.length,
addableJobCount,
});
- const jobs = univistedUrls.slice(0, addableJobCount).map((url) => {
+ const jobs = univistedUrls.links.slice(0, addableJobCount).map((url) => {
const uuid = uuidv4();
return {
name: uuid,
@@ -949,12 +949,13 @@ async function kickoffGetIndexLinks(sc: StoredCrawl, crawler: WebCrawler, url: s
sc.crawlerOptions.limit ?? 10000,
);
- const validIndexLinks = crawler.filterLinks(
- index.filter(x => crawler.filterURL(x, trimmedURL.href) !== null),
+ const validIndexLinksResult = crawler.filterLinks(
+ index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),
sc.crawlerOptions.limit ?? 10000,
sc.crawlerOptions.maxDepth ?? 10,
false,
);
+ const validIndexLinks = validIndexLinksResult.links;
return validIndexLinks;
}
@@ -1308,14 +1309,10 @@ async function processJob(job: Job & { id: string }, token: string) {
job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
) {
const crawler = crawlToCrawler(job.data.crawl_id, sc, (await getACUCTeam(job.data.team_id))?.flags ?? null);
- if (
- crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL) ===
- null &&
- !job.data.isCrawlSourceScrape
- ) {
- throw new Error(
- "Redirected target URL is not allowed by crawlOptions",
- ); // TODO: make this its own error type that is ignored by error tracking
+ const filterResult = crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL);
+ if (!filterResult.allowed && !job.data.isCrawlSourceScrape) {
+ const reason = filterResult.denialReason || "Redirected target URL is not allowed by crawlOptions";
+ throw new Error(reason);
}
// Only re-set originUrl if it's different from the current hostname
@@ -1379,11 +1376,11 @@ async function processJob(job: Job & { id: string }, token: string) {
Infinity,
sc.crawlerOptions?.maxDepth ?? 10,
);
- logger.debug("Discovered " + links.length + " links...", {
- linksLength: links.length,
+ logger.debug("Discovered " + links.links.length + " links...", {
+ linksLength: links.links.length,
});
- for (const link of links) {
+ for (const link of links.links) {
if (await lockURL(job.data.crawl_id, sc, link)) {
// This seems to work really welel
const jobPriority = await getJobPriority({
@@ -1442,17 +1439,17 @@ async function processJob(job: Job & { id: string }, token: string) {
}
// Only run check after adding new jobs for discovery - mogery
- if (
- job.data.isCrawlSourceScrape &&
- crawler.filterLinks(
+ if (job.data.isCrawlSourceScrape) {
+ const filterResult = crawler.filterLinks(
[doc.metadata.url ?? doc.metadata.sourceURL!],
1,
sc.crawlerOptions?.maxDepth ?? 10,
- ).length === 0
- ) {
- throw new Error(
- "Source URL is not allowed by includePaths/excludePaths rules",
);
+ if (filterResult.links.length === 0) {
+ const url = doc.metadata.url ?? doc.metadata.sourceURL!;
+ const reason = filterResult.denialReasons.get(url) || "Source URL is not allowed by crawl configuration";
+ throw new Error(reason);
+ }
}
}
}
@@ -1701,4 +1698,4 @@ app.listen(3005, () => {
console.log("All jobs finished. Worker out!");
process.exit(0);
-})();
\ No newline at end of file
+})();