Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
added trycatch and removed redundancy
rafaelmmillerf07bbef78e4a
diff --git a/apps/api/src/scraper/WebScraper/utils/utils.ts b/apps/api/src/scraper/WebScraper/utils/utils.tsindex 872adc6e5..849d209de 100644--- a/apps/api/src/scraper/WebScraper/utils/utils.ts+++ b/apps/api/src/scraper/WebScraper/utils/utils.ts@@ -29,27 +29,28 @@ export function extractLinks(html: string, baseUrl: string): string[] {const $ = cheerio.load(html);const links: string[] = [];- // Parse the base URL to get the origin- const urlObject = new URL(baseUrl);- const origin = urlObject.origin;-$('a').each((_, element) => {const href = $(element).attr('href');if (href) {- if (href.startsWith('http://') || href.startsWith('https://')) {- // Absolute URL, add as is- links.push(href);- } else if (href.startsWith('/')) {- // Relative URL starting with '/', append to origin- links.push(new URL(href, baseUrl).href);- } else if (!href.startsWith('#') && !href.startsWith('mailto:')) {- // Relative URL not starting with '/', append to base URL- links.push(new URL(href, baseUrl).href);- } else if (href.startsWith('mailto:')) {- // mailto: links, add as is- links.push(href);+ try {+ if (href.startsWith('http://') || href.startsWith('https://')) {+ // Absolute URL, add as is+ links.push(href);+ } else if (href.startsWith('/')) {+ // Relative URL starting with '/', append to base URL+ links.push(new URL(href, baseUrl).href);+ } else if (!href.startsWith('#') && !href.startsWith('mailto:')) {+ // Relative URL not starting with '/', append to base URL+ links.push(new URL(href, baseUrl).href);+ } else if (href.startsWith('mailto:')) {+ // mailto: links, add as is+ links.push(href);+ }+ // Fragment-only links (#) are ignored+ } catch (error) {+ // Log the error and continue+ console.error(`Failed to construct URL for href: ${href} with base: ${baseUrl}`, error);}- // Fragment-only links (#) are ignored}});