Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add options.baseUrl to enqueueLinks

Ondra Urbanca112ed52a91
diff --git a/src/enqueue_links.js b/src/enqueue_links.js
index 12c8e3d42..221e4bea5 100644
--- a/src/enqueue_links.js
+++ b/src/enqueue_links.js
@@ -1,3 +1,4 @@
+import { URL } from 'url';
import _ from 'underscore';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
@@ -64,11 +65,26 @@ export const extractUrlsFromPage = async (page, selector) => {
*
* @param {Function} $
* @param {string} selector
+ * @param {string} baseUrl
* @returns {string[]}
* @ignore
*/
-export const extraxtUrlsFromCheerio = ($, selector) => {
- return $(selector).map((i, el) => $(el).attr('href')).get().filter(href => !!href);
+export const extraxtUrlsFromCheerio = ($, selector, baseUrl) => {
+ return $(selector)
+ .map((i, el) => $(el).attr('href'))
+ .get()
+ .filter(href => href)
+ .map((href) => {
+ // Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
+ const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
+ if (!isHrefAbsolute && !baseUrl) {
+ throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
+ + 'Use options.baseUrl in utils.enqueueLinks() to automatically resolve relative URLs.');
+ }
+ return baseUrl
+ ? (new URL(href, baseUrl)).href
+ : href;
+ });
};
/**
@@ -122,6 +138,9 @@ let logDeprecationWarning = true;
* A request queue to which the URLs will be enqueued.
* @param {String} [options.selector='a']
* A CSS selector matching links to be enqueued.
+ * @param {string} [options.baseUrl]
+ * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+ * since the relative URL resolution is done inside the browser automatically.
* @param {Object[]|String[]} [options.pseudoUrls]
* An array of {@link PseudoUrl}s matching the URLs to be enqueued,
* or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
@@ -168,9 +187,9 @@ let logDeprecationWarning = true;
export const enqueueLinks = async (...args) => {
// TODO: Remove after v1.0.0 gets released.
// Refactor enqueueLinks to use an options object and keep backwards compatibility
- let page, $, selector, requestQueue, pseudoUrls, userData; // eslint-disable-line
+ let page, $, selector, requestQueue, baseUrl, pseudoUrls, userData; // eslint-disable-line
if (args.length === 1) {
- [{ page, $, selector = 'a', requestQueue, pseudoUrls, userData = {} }] = args;
+ [{ page, $, selector = 'a', requestQueue, baseUrl, pseudoUrls, userData = {} }] = args;
} else {
[page, selector = 'a', requestQueue, pseudoUrls, userData = {}] = args;
if (logDeprecationWarning) {
@@ -197,13 +216,15 @@ export const enqueueLinks = async (...args) => {
}
checkParamOrThrow(selector, 'selector', 'String');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
+ checkParamOrThrow(baseUrl, 'baseUrl', 'Maybe String');
+ if (baseUrl && page) log.warning('The parameter options.baseUrl can only be used when parsing a Cheerio object. It will be ignored.');
checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');
checkParamOrThrow(userData, 'userData', 'Maybe Object');
// Construct pseudoUrls from input where necessary.
const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);
- const urls = page ? await extractUrlsFromPage(page, selector) : extraxtUrlsFromCheerio($, selector);
+ const urls = page ? await extractUrlsFromPage(page, selector) : extraxtUrlsFromCheerio($, selector, baseUrl);
let requests = [];
if (pseudoUrlInstances.length) {
diff --git a/test/enqueue_links.js b/test/enqueue_links.js
index 0a5576264..1fbcc35b0 100644
--- a/test/enqueue_links.js
+++ b/test/enqueue_links.js
@@ -6,6 +6,30 @@ import { RequestQueue } from '../build/request_queue';
const { utils: { log } } = Apify;
+const HTML = `
+<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+ </p>
+ <ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
+ <p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+ </p>
+ <a href="/x/relative">This is a relative link.</a>
+ </body>
+</html>
+`;
+
describe('enqueueLinks()', () => {
let ll;
before(() => {
@@ -24,26 +48,7 @@ describe('enqueueLinks()', () => {
beforeEach(async () => {
browser = await Apify.launchPuppeteer({ headless: true });
page = await browser.newPage();
- await page.setContent(`<html>
- <head>
- <title>Example</title>
- </head>
- <body>
- <p>
- The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
- </p>
- <ul>
- <li>These aren't the Droids you're looking for</li>
- <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
- <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
- </ul>
- <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
- <p>
- Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
- just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
- </p>
- </body>
- </html>`);
+ await page.setContent(HTML);
});
afterEach(async () => {
@@ -336,26 +341,7 @@ describe('enqueueLinks()', () => {
let $;
beforeEach(async () => {
- $ = cheerio.load(`<html>
- <head>
- <title>Example</title>
- </head>
- <body>
- <p>
- The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
- </p>
- <ul>
- <li>These aren't the Droids you're looking for</li>
- <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
- <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
- </ul>
- <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
- <p>
- Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
- just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
- </p>
- </body>
- </html>`);
+ $ = cheerio.load(HTML);
});
afterEach(async () => {
@@ -606,5 +592,56 @@ describe('enqueueLinks()', () => {
expect(enqueued).to.have.lengthOf(0);
}
});
+
+ it('correctly resolves relative URLs', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ $, requestQueue, baseUrl: 'http://www.absolute.com/removethis' });
+
+ expect(enqueued).to.have.lengthOf(6);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/second');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+
+ expect(enqueued[4].url).to.be.eql('http://cool.com/');
+ expect(enqueued[4].method).to.be.eql('GET');
+ expect(enqueued[4].userData).to.be.eql({});
+
+ expect(enqueued[5].url).to.be.eql('http://www.absolute.com/x/relative');
+ expect(enqueued[5].method).to.be.eql('GET');
+ expect(enqueued[5].userData).to.be.eql({});
+ });
+
+ it('throws on finding a relative link with no baseUrl set', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ try {
+ await enqueueLinks({ $, requestQueue });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).to.include('/x/relative');
+ }
+ expect(enqueued).to.have.lengthOf(0);
+ });
});
});