Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix unreachable allowSubdomains code in crawler filterURL method (#1719)

devin-ai-integration[bot]070d1c1d9818
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index 3a565fc52..df85c29c3 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -150,4 +150,33 @@ describe("Crawl tests", () => {
expect(res.completed).toBeGreaterThan(0);
}
}, 5 * scrapeTimeout);
+
+ it.concurrent("allowSubdomains parameter works", async () => {
+ const res = await crawl({
+ url: "https://firecrawl.dev",
+ allowSubdomains: true,
+ limit: 5,
+ }, identity);
+
+ expect(res.success).toBe(true);
+ if (res.success) {
+ expect(res.completed).toBeGreaterThan(0);
+ }
+ }, 5 * scrapeTimeout);
+
+ it.concurrent("allowSubdomains blocks subdomains when false", async () => {
+ const res = await crawl({
+ url: "https://firecrawl.dev",
+ allowSubdomains: false,
+ limit: 5,
+ }, identity);
+
+ expect(res.success).toBe(true);
+ if (res.success) {
+ for (const page of res.data) {
+ const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);
+ expect(url.hostname.endsWith("firecrawl.dev")).toBe(true);
+ }
+ }
+ }, 5 * scrapeTimeout);
});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 90fd9afe5..d5cf8f1f5 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -459,18 +459,16 @@ export class WebCrawler {
return { allowed: true, url: fullUrl };
}
+ if (
+ this.allowSubdomains &&
+ !this.isSocialMediaOrEmail(fullUrl) &&
+ this.isSubdomain(fullUrl)
+ ) {
+ return { allowed: true, url: fullUrl };
+ }
+
return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };
}
-
- if (
- this.allowSubdomains &&
- !this.isSocialMediaOrEmail(fullUrl) &&
- this.isSubdomain(fullUrl)
- ) {
- return { allowed: true, url: fullUrl };
- }
-
- return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };
}
private async extractLinksFromHTMLRust(html: string, url: string) {