Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: use PSL
amplitudes3c2e1d26bc97
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.tsindex 650c881c3..d566c34f1 100644--- a/apps/api/src/__tests__/snips/v2/map.test.ts+++ b/apps/api/src/__tests__/snips/v2/map.test.ts@@ -100,14 +100,15 @@ describe("Map tests", () => {expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);- // If we get ≤ 1 results and the URL is not a base domain, we should get a warning- if (response.body.links.length <= 1) {- expect(response.body.warning).toBeDefined();- expect(response.body.warning).toContain("Only");- expect(response.body.warning).toContain("result(s) found");- expect(response.body.warning).toContain("base domain");- expect(response.body.warning).toContain("example.com");- }+ // Assert the prerequisite condition+ expect(response.body.links.length).toBeLessThanOrEqual(1);++ // Check that the warning is present+ expect(response.body.warning).toBeDefined();+ expect(response.body.warning).toContain("Only");+ expect(response.body.warning).toContain("result(s) found");+ expect(response.body.warning).toContain("base domain");+ expect(response.body.warning).toContain("example.com");},60000,);diff --git a/apps/api/src/lib/url-utils.ts b/apps/api/src/lib/url-utils.tsindex d46ae24db..4d9ccc8db 100644--- a/apps/api/src/lib/url-utils.ts+++ b/apps/api/src/lib/url-utils.ts@@ -2,6 +2,8 @@* Utility functions for URL manipulation and analysis*/+import psl from "psl";+/*** Determines if a URL is a base domain (e.g., example.com vs blog.example.com or example.com/path)* A base domain is considered to be the root domain without subdomains or paths@@ -13,14 +15,18 @@ export function isBaseDomain(url: string): boolean {const urlObj = new URL(url);const hostname = urlObj.hostname;- // Remove www. prefix for consistency+ // Parse the domain using psl+ const parsed = psl.parse(hostname);+ if (!parsed.domain) {+ return false;+ }++ // Check if hostname equals the domain (no subdomain)const cleanHostname = hostname.startsWith("www.")? hostname.slice(4): hostname;- // Check if there are subdomains (more than 2 parts when split by dots)- const parts = cleanHostname.split(".");- if (parts.length > 2) {+ if (cleanHostname !== parsed.domain) {return false; // Has subdomains}@@ -47,43 +53,9 @@ export function extractBaseDomain(url: string): string | null {const urlObj = new URL(url);const hostname = urlObj.hostname;- // Remove www. prefix for consistency- const cleanHostname = hostname.startsWith("www.")- ? hostname.slice(4)- : hostname;-- // Extract the base domain- const parts = cleanHostname.split(".");- if (parts.length >= 2) {- // Handle special cases for multi-part TLDs like .co.uk, .com.au, etc.- const lastTwoParts = parts.slice(-2).join(".");-- // Common multi-part TLDs that should be treated as single TLD- const multiPartTlds = [- "co.uk",- "com.au",- "org.uk",- "net.uk",- "ac.uk",- "gov.uk",- "co.nz",- "com.br",- "co.jp",- "co.kr",- "co.in",- "co.za",- ];-- if (multiPartTlds.includes(lastTwoParts) && parts.length >= 3) {- // For domains like subdomain.example.co.uk, return example.co.uk- return parts.slice(-3).join(".");- } else {- // For regular domains like example.com, return the last 2 parts- return lastTwoParts;- }- }+ const parsed = psl.parse(hostname);- return cleanHostname;+ return parsed.domain || null;} catch (error) {return null;}diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex b86cbf355..6e5c67065 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -3,7 +3,6 @@ import { load } from "cheerio"; // rustifiedimport { URL } from "url";import { getLinksFromSitemap } from "./sitemap";import robotsParser, { Robot } from "robots-parser";-import psl from "psl";import { getURLDepth } from "./utils/maxDepthUtils";import { logger as _logger } from "../../lib/logger";import { redisEvictConnection } from "../../services/redis";