Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: use PSL

amplitudes3c2e1d26bc97
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.ts
index 650c881c3..d566c34f1 100644
--- a/apps/api/src/__tests__/snips/v2/map.test.ts
+++ b/apps/api/src/__tests__/snips/v2/map.test.ts
@@ -100,14 +100,15 @@ describe("Map tests", () => {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
- // If we get ≤ 1 results and the URL is not a base domain, we should get a warning
- if (response.body.links.length <= 1) {
- expect(response.body.warning).toBeDefined();
- expect(response.body.warning).toContain("Only");
- expect(response.body.warning).toContain("result(s) found");
- expect(response.body.warning).toContain("base domain");
- expect(response.body.warning).toContain("example.com");
- }
+ // Assert the prerequisite condition
+ expect(response.body.links.length).toBeLessThanOrEqual(1);
+
+ // Check that the warning is present
+ expect(response.body.warning).toBeDefined();
+ expect(response.body.warning).toContain("Only");
+ expect(response.body.warning).toContain("result(s) found");
+ expect(response.body.warning).toContain("base domain");
+ expect(response.body.warning).toContain("example.com");
},
60000,
);
diff --git a/apps/api/src/lib/url-utils.ts b/apps/api/src/lib/url-utils.ts
index d46ae24db..4d9ccc8db 100644
--- a/apps/api/src/lib/url-utils.ts
+++ b/apps/api/src/lib/url-utils.ts
@@ -2,6 +2,8 @@
* Utility functions for URL manipulation and analysis
*/
+import psl from "psl";
+
/**
* Determines if a URL is a base domain (e.g., example.com vs blog.example.com or example.com/path)
* A base domain is considered to be the root domain without subdomains or paths
@@ -13,14 +15,18 @@ export function isBaseDomain(url: string): boolean {
const urlObj = new URL(url);
const hostname = urlObj.hostname;
- // Remove www. prefix for consistency
+ // Parse the domain using psl
+ const parsed = psl.parse(hostname);
+ if (!parsed.domain) {
+ return false;
+ }
+
+ // Check if hostname equals the domain (no subdomain)
const cleanHostname = hostname.startsWith("www.")
? hostname.slice(4)
: hostname;
- // Check if there are subdomains (more than 2 parts when split by dots)
- const parts = cleanHostname.split(".");
- if (parts.length > 2) {
+ if (cleanHostname !== parsed.domain) {
return false; // Has subdomains
}
@@ -47,43 +53,9 @@ export function extractBaseDomain(url: string): string | null {
const urlObj = new URL(url);
const hostname = urlObj.hostname;
- // Remove www. prefix for consistency
- const cleanHostname = hostname.startsWith("www.")
- ? hostname.slice(4)
- : hostname;
-
- // Extract the base domain
- const parts = cleanHostname.split(".");
- if (parts.length >= 2) {
- // Handle special cases for multi-part TLDs like .co.uk, .com.au, etc.
- const lastTwoParts = parts.slice(-2).join(".");
-
- // Common multi-part TLDs that should be treated as single TLD
- const multiPartTlds = [
- "co.uk",
- "com.au",
- "org.uk",
- "net.uk",
- "ac.uk",
- "gov.uk",
- "co.nz",
- "com.br",
- "co.jp",
- "co.kr",
- "co.in",
- "co.za",
- ];
-
- if (multiPartTlds.includes(lastTwoParts) && parts.length >= 3) {
- // For domains like subdomain.example.co.uk, return example.co.uk
- return parts.slice(-3).join(".");
- } else {
- // For regular domains like example.com, return the last 2 parts
- return lastTwoParts;
- }
- }
+ const parsed = psl.parse(hostname);
- return cleanHostname;
+ return parsed.domain || null;
} catch (error) {
return null;
}
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index b86cbf355..6e5c67065 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -3,7 +3,6 @@ import { load } from "cheerio"; // rustified
import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import robotsParser, { Robot } from "robots-parser";
-import psl from "psl";
import { getURLDepth } from "./utils/maxDepthUtils";
import { logger as _logger } from "../../lib/logger";
import { redisEvictConnection } from "../../services/redis";