Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix unreachable allowSubdomains code in crawler filterURL method (#1719)
devin-ai-integration[bot]070d1c1d9818
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 3a565fc52..df85c29c3 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -150,4 +150,33 @@ describe("Crawl tests", () => {expect(res.completed).toBeGreaterThan(0);}}, 5 * scrapeTimeout);++ it.concurrent("allowSubdomains parameter works", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ allowSubdomains: true,+ limit: 5,+ }, identity);++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.completed).toBeGreaterThan(0);+ }+ }, 5 * scrapeTimeout);++ it.concurrent("allowSubdomains blocks subdomains when false", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ allowSubdomains: false,+ limit: 5,+ }, identity);++ expect(res.success).toBe(true);+ if (res.success) {+ for (const page of res.data) {+ const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);+ expect(url.hostname.endsWith("firecrawl.dev")).toBe(true);+ }+ }+ }, 5 * scrapeTimeout);});diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 90fd9afe5..d5cf8f1f5 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -459,18 +459,16 @@ export class WebCrawler {return { allowed: true, url: fullUrl };}+ if (+ this.allowSubdomains &&+ !this.isSocialMediaOrEmail(fullUrl) &&+ this.isSubdomain(fullUrl)+ ) {+ return { allowed: true, url: fullUrl };+ }+return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };}-- if (- this.allowSubdomains &&- !this.isSocialMediaOrEmail(fullUrl) &&- this.isSubdomain(fullUrl)- ) {- return { allowed: true, url: fullUrl };- }-- return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };}private async extractLinksFromHTMLRust(html: string, url: string) {