Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl): relative URL page discovery issues
Móricz Gergő173028295b7c
diff --git a/apps/api/sharedLibs/html-transformer/src/lib.rs b/apps/api/sharedLibs/html-transformer/src/lib.rsindex f5eb86e50..d4a299342 100644--- a/apps/api/sharedLibs/html-transformer/src/lib.rs+++ b/apps/api/sharedLibs/html-transformer/src/lib.rs@@ -17,7 +17,8 @@ pub unsafe extern "C" fn extract_links(html: *const libc::c_char) -> *mut i8 {let mut out: Vec<String> = Vec::new();- for anchor in document.select("a[href]").unwrap() {+ let anchors: Vec<_> = document.select("a[href]").unwrap().collect();+ for anchor in anchors {let mut href = anchor.attributes.borrow().get("href").unwrap().to_string();if href.starts_with("http:/") && !href.starts_with("http://") {diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex ceee94efc..ea606f440 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -401,7 +401,13 @@ export class WebCrawler {public async extractLinksFromHTML(html: string, url: string) {try {- return await this.extractLinksFromHTMLRust(html, url);+ return (await this.extractLinksFromHTMLRust(html, url)).map(x => {+ try {+ return new URL(x, url).href+ } catch (e) {+ return null;+ }+ }).filter(x => x !== null) as string[];} catch (error) {this.logger.error("Failed to call html-transformer! Falling back to cheerio...", {error,