Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: update utils.enqueueLinks to ignore invalid URLs when using cheerio (#1369)
Ashley Evans953fb66c07b6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex dab11a693..accca765d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,7 @@2.3.3 / TBA====================- fix: update `purgeLocalStorage` utility method to mimic `purge` param of `apify run`+- fix: update `utils.enqueueLinks` to ignore invalid URLs when using cheerio2.3.2 / 2022/05/05====================diff --git a/src/enqueue_links/enqueue_links.js b/src/enqueue_links/enqueue_links.jsindex 824a164fb..4390e9b99 100644--- a/src/enqueue_links/enqueue_links.js+++ b/src/enqueue_links/enqueue_links.js@@ -171,15 +171,20 @@ export function extractUrlsFromCheerio($, selector, baseUrl) {.map((i, el) => $(el).attr('href')).get().filter((href) => !!href)- .map((href) => {+ .reduce((urls, current) => {// Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.- const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.+ const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(current); // Grabbed this in 'is-absolute-url' package.if (!isHrefAbsolute && !baseUrl) {- throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `+ throw new Error(`An extracted URL: ${current} is relative and options.baseUrl is not set. `+ 'Use options.baseUrl in utils.enqueueLinks() to automatically resolve relative URLs.');}- return baseUrl- ? (new URL(href, baseUrl)).href- : href;- });++ try {+ urls.push(new URL(current, baseUrl).href);+ } catch {+ log.warning(`An extracted URL: ${current} is invalid, ignoring.`);+ }++ return urls;+ }, []);}diff --git a/test/enqueue_links/enqueue_links.test.js b/test/enqueue_links/enqueue_links.test.jsindex 828d4109a..19bb04792 100644--- a/test/enqueue_links/enqueue_links.test.js+++ b/test/enqueue_links/enqueue_links.test.js@@ -31,6 +31,30 @@ const HTML = `</html>`;+const HTML_WITH_INVALID_LINK = `+<html>+ <head>+ <title>Example</title>+ </head>+ <body>+ <a href="https://invalid url 1">Invalid!</a>+ <a href="https://www.completelyvalid.com/">Valid!</a>+ </body>+</html>+`;++const HTML_WITH_INVALID_RELATIVE_LINKS = `+<html>+ <head>+ <title>Example</title>+ </head>+ <body>+ <a href="////"</a>+ <a href="/validrelativeurl">Valid!</a>+ </body>+</html>+`;+describe('enqueueLinks()', () => {let ll;beforeAll(() => {@@ -615,5 +639,47 @@ describe('enqueueLinks()', () => {}expect(enqueued).toHaveLength(0);});++ test('ignores invalid URLs', async () => {+ $ = cheerio.load(HTML_WITH_INVALID_LINK);+ const enqueued = [];+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };++ await enqueueLinks({ $, requestQueue });++ expect(enqueued).toHaveLength(1);+ expect(enqueued[0].url).toBe('https://www.completelyvalid.com/');+ });++ test('ignores invalid URLs if base URL set', async () => {+ $ = cheerio.load(HTML_WITH_INVALID_LINK);+ const enqueued = [];+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };++ await enqueueLinks({ $, requestQueue, baseUrl: 'https://www.example.com/' });++ expect(enqueued).toHaveLength(1);+ expect(enqueued[0].url).toBe('https://www.completelyvalid.com/');+ });++ test('ignores invalid relative URLs if base URL set', async () => {+ $ = cheerio.load(HTML_WITH_INVALID_RELATIVE_LINKS);+ const enqueued = [];+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });+ requestQueue.addRequest = async (request) => {+ enqueued.push(request);+ };++ await enqueueLinks({ $, requestQueue, baseUrl: 'https://www.example.com/' });++ expect(enqueued).toHaveLength(1);+ expect(enqueued[0].url).toBe('https://www.example.com/validrelativeurl');+ });});});