Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: ignore invalid URLs in `enqueueLinks` in browser crawlers (#1803)
robertm-975ac336c5b83b
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex ebc25ec2a..9bdcc1360 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -23,6 +23,7 @@ import {BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,BasicCrawler,RequestState,+ tryAbsoluteURL,} from '@crawlee/basic';import type {BrowserController,@@ -723,8 +724,10 @@ async function extractUrlsFromPage(page: { $$eval: Function }, selector: string,throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `+ 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');}+return baseUrl- ? (new URL(href, baseUrl)).href+ ? tryAbsoluteURL(href, baseUrl): href;- });+ })+ .filter((href: string | undefined) => !!href);}diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 9f6dfba83..7bd781772 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -8,7 +8,7 @@ import type {RequestQueue,Configuration,} from '@crawlee/http';-import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';+import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';import type { CheerioOptions } from 'cheerio';import * as cheerio from 'cheerio';@@ -224,15 +224,8 @@ function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrlthrow new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `+ 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');}- const tryAbsolute = () => {- try {- return (new URL(href, baseUrl)).href;- } catch {- return undefined;- }- };return baseUrl- ? tryAbsolute()+ ? tryAbsoluteURL(href, baseUrl): href;}).filter((href) => !!href) as string[];diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 7ac577fc2..50ebd601c 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -211,6 +211,17 @@ export function createRequestOptions(});}+/**+ * Helper function used to validate URLs used when extracting URLs from a page+ */+export function tryAbsoluteURL(href: string, baseUrl: string): string | undefined {+ try {+ return (new URL(href, baseUrl)).href;+ } catch {+ return undefined;+ }+}+/*** Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used* {@apilink enqueueLinks} to modify requests before enqueuing them.diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 977964fa5..a26c9a31c 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -9,7 +9,7 @@ import type {RequestQueue,Configuration,} from '@crawlee/http';-import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';+import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { concatStreamToBuffer } from '@apify/utilities';import type { DOMWindow } from 'jsdom';@@ -265,12 +265,7 @@ function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: strif (href === undefined) {return undefined;}-- try {- return (new URL(href, baseUrl)).href;- } catch {- return undefined;- }+ return tryAbsoluteURL(href, baseUrl);}).filter((href) => href !== undefined && href !== '') as string[];}diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex c3d3eb16a..69fb03f0d 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -38,6 +38,7 @@ const HTML = `<a href="/x/absolutepath">This is a relative link.</a><a href="y/relativepath">This is a relative link.</a><a href="//example.absolute.com/hello">This is a link to a different subdomain</a>+ <a href="http://">Invalid URL link, this needs to be ignored</a></body></html>`;