Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
ENG-2829: Fix isSubdomain bug (#1845)
devin-ai-integration[bot]104b6327b221
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 93bd96d04..cccbb7370 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -210,6 +210,29 @@ describe("Crawl tests", () => {}}, 5 * scrapeTimeout);+ it.concurrent("allowSubdomains correctly allows same registrable domain using PSL", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ allowSubdomains: true,+ allowExternalLinks: false,+ limit: 3,+ }, identity);++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.data.length).toBeGreaterThan(0);+ for (const page of res.data) {+ const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);+ const hostname = url.hostname;++ expect(+ hostname === "firecrawl.dev" ||+ hostname.endsWith(".firecrawl.dev")+ ).toBe(true);+ }+ }+ }, 5 * scrapeTimeout);+it.concurrent("rejects crawl when URL depth exceeds maxDepth", async () => {const response = await crawlStart({url: "https://firecrawl.dev/blog/category/deep/nested/path",diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 9e5adaccf..d6fcda657 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -3,6 +3,7 @@ import { load } from "cheerio"; // rustifiedimport { URL } from "url";import { getLinksFromSitemap } from "./sitemap";import robotsParser, { Robot } from "robots-parser";+import psl from "psl";import { getURLDepth } from "./utils/maxDepthUtils";import { axiosTimeout } from "../../lib/timeout";import { logger as _logger } from "../../lib/logger";@@ -653,9 +654,21 @@ export class WebCrawler {}private isSubdomain(link: string): boolean {- return new URL(link, this.baseUrl).hostname.endsWith(- "." + new URL(this.baseUrl).hostname.split(".").slice(-2).join("."),- );+ try {+ const linkUrl = new URL(link, this.baseUrl);+ const baseUrl = new URL(this.baseUrl);++ const linkParsed = psl.parse(linkUrl.hostname);+ const baseParsed = psl.parse(baseUrl.hostname);++ if (!linkParsed?.domain || !baseParsed?.domain) {+ return false;+ }++ return linkParsed.domain === baseParsed.domain;+ } catch (error) {+ return false;+ }}public isFile(url: string): boolean {