Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix 2

Gergő Móriczb1eaecfdb0f1
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index cb93eedea..a7c7de753 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -233,7 +233,7 @@ export class WebCrawler {
let href = $(element).attr("href");
if (href) {
if (href.match(/^https?:\/[^\/]/)) {
- href = href.replace(/^https?:\/[^\/]/, "$&/");
+ href = href.replace(/^https?:\//, "$&/");
}
const u = this.filterURL(href, url);
if (u !== null) {