Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix handling of badly formatted URLs
Gergő Móricze2ddc6c65c08
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 3828f8302..cb93eedea 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -230,8 +230,11 @@ export class WebCrawler {const $ = load(html);$("a").each((_, element) => {- const href = $(element).attr("href");+ let href = $(element).attr("href");if (href) {+ if (href.match(/^https?:\/[^\/]/)) {+ href = href.replace(/^https?:\/[^\/]/, "$&/");+ }const u = this.filterURL(href, url);if (u !== null) {links.push(u);diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex db9769270..c75acd322 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -352,12 +352,10 @@ async function processJob(job: Job & { id: string }, token: string) {if (job.data.crawlerOptions !== null) {if (!sc.cancelled) {- const newURL = new URL(doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!);- const useNewURLAsBase = newURL.hostname.split(".").slice(-2).join(".") === new URL(sc.originUrl!).hostname.split(".").slice(-2).join(".");- const crawler = crawlToCrawler(job.data.crawl_id, sc, useNewURLAsBase ? newURL.href : undefined);+ const crawler = crawlToCrawler(job.data.crawl_id, sc, doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!);const links = crawler.filterLinks(- crawler.extractLinksFromHTML(rawHtml ?? "", doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl as string),+ crawler.extractLinksFromHTML(rawHtml ?? "", doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!),Infinity,sc.crawlerOptions?.maxDepth ?? 10);