Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): decode percent-encoded hosts in the blocklist check (#5654)

tomsideguideea9e83d4cec7
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index c0be26e70..f407a33f9 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -236,6 +236,10 @@ function findBlockedMatch(
let parsedUrl: any;
try {
parsedUrl = parse(decryptedUrl);
+ // tldts doesn't percent-decode hosts the way fetchers do.
+ if (!parsedUrl.domain && URL.canParse(decryptedUrl)) {
+ parsedUrl = parse(new URL(decryptedUrl).hostname);
+ }
} catch {
console.log("Error parsing URL:", url);
return null;