Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix 2
Gergő Móriczb1eaecfdb0f1
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex cb93eedea..a7c7de753 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -233,7 +233,7 @@ export class WebCrawler {let href = $(element).attr("href");if (href) {if (href.match(/^https?:\/[^\/]/)) {- href = href.replace(/^https?:\/[^\/]/, "$&/");+ href = href.replace(/^https?:\//, "$&/");}const u = this.filterURL(href, url);if (u !== null) {