Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

ENG-2829: Fix isSubdomain bug (#1845)

devin-ai-integration[bot]104b6327b221
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index 93bd96d04..cccbb7370 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -210,6 +210,29 @@ describe("Crawl tests", () => {
}
}, 5 * scrapeTimeout);
+ it.concurrent("allowSubdomains correctly allows same registrable domain using PSL", async () => {
+ const res = await crawl({
+ url: "https://firecrawl.dev",
+ allowSubdomains: true,
+ allowExternalLinks: false,
+ limit: 3,
+ }, identity);
+
+ expect(res.success).toBe(true);
+ if (res.success) {
+ expect(res.data.length).toBeGreaterThan(0);
+ for (const page of res.data) {
+ const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);
+ const hostname = url.hostname;
+
+ expect(
+ hostname === "firecrawl.dev" ||
+ hostname.endsWith(".firecrawl.dev")
+ ).toBe(true);
+ }
+ }
+ }, 5 * scrapeTimeout);
+
it.concurrent("rejects crawl when URL depth exceeds maxDepth", async () => {
const response = await crawlStart({
url: "https://firecrawl.dev/blog/category/deep/nested/path",
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 9e5adaccf..d6fcda657 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -3,6 +3,7 @@ import { load } from "cheerio"; // rustified
import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import robotsParser, { Robot } from "robots-parser";
+import psl from "psl";
import { getURLDepth } from "./utils/maxDepthUtils";
import { axiosTimeout } from "../../lib/timeout";
import { logger as _logger } from "../../lib/logger";
@@ -653,9 +654,21 @@ export class WebCrawler {
}
private isSubdomain(link: string): boolean {
- return new URL(link, this.baseUrl).hostname.endsWith(
- "." + new URL(this.baseUrl).hostname.split(".").slice(-2).join("."),
- );
+ try {
+ const linkUrl = new URL(link, this.baseUrl);
+ const baseUrl = new URL(this.baseUrl);
+
+ const linkParsed = psl.parse(linkUrl.hostname);
+ const baseParsed = psl.parse(baseUrl.hostname);
+
+ if (!linkParsed?.domain || !baseParsed?.domain) {
+ return false;
+ }
+
+ return linkParsed.domain === baseParsed.domain;
+ } catch (error) {
+ return false;
+ }
}
public isFile(url: string): boolean {