Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): decode percent-encoded hosts in the blocklist check (#5654)
tomsideguideea9e83d4cec7
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex c0be26e70..f407a33f9 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -236,6 +236,10 @@ function findBlockedMatch(let parsedUrl: any;try {parsedUrl = parse(decryptedUrl);+ // tldts doesn't percent-decode hosts the way fetchers do.+ if (!parsedUrl.domain && URL.canParse(decryptedUrl)) {+ parsedUrl = parse(new URL(decryptedUrl).hostname);+ }} catch {console.log("Error parsing URL:", url);return null;