Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed Apify.utils.puppeteer.enqueueLinks to allow null value for pseudoUrls param

Jan Curn396574d3e920
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 06ca29f44..d3a91b560 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxx
+===================
+- Fixed `Apify.utils.puppeteer.enqueueLinks` to allow `null` value for `pseudoUrls` param
+
0.8.19 / 2018-11-05
===================
- Fixed `RequestQueue.addRequest()` to gracefully handle invalid URLs
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 12fdbcdc4..ec6f53e5c 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -197,11 +197,13 @@ let logDeprecationWarning = true;
* or an array of Strings or Objects from which the {@link PseudoUrl}s should be constructed
* The Objects must include at least a `purl` property, which holds a pseudoUrl string.
* All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor.
+ * If `pseudoUrls` is an empty array, null or undefined, then the function
+ * enqueues all links found on the page.
* @return {Promise<QueueOperationInfo[]>}
* Promise that resolves to an array of {@link QueueOperationInfo} objects.
* @memberOf puppeteer
*/
-const enqueueLinks = async (page, selector, requestQueue, pseudoUrls = []) => {
+const enqueueLinks = async (page, selector, requestQueue, pseudoUrls) => {
// TODO: Remove after v1.0.0 gets released.
// Check for pseudoUrls as a third parameter.
if (Array.isArray(requestQueue)) {
@@ -218,10 +220,10 @@ const enqueueLinks = async (page, selector, requestQueue, pseudoUrls = []) => {
checkParamOrThrow(page, 'page', 'Object');
checkParamOrThrow(selector, 'selector', 'String');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
- checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Array');
+ checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');
// Construct pseudoUrls from input where necessary.
- const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls);
+ const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);
/* istanbul ignore next */
const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 316f50a74..dff3ff833 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -209,7 +209,7 @@ describe('Apify.utils.puppeteer', () => {
expect(enqueued[2].userData).to.be.eql({});
});
- it('works with empty pseudoUrls[]', async () => {
+ it('works with undefined pseudoUrls[]', async () => {
const enqueued = [];
const queue = new RequestQueue('xxx');
queue.addRequest = async (request) => {
@@ -236,6 +236,63 @@ describe('Apify.utils.puppeteer', () => {
expect(enqueued[3].method).to.be.eql(undefined);
expect(enqueued[3].userData).to.be.eql(undefined);
});
+
+ it('works with null pseudoUrls[]', async () => {
+ const enqueued = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, null);
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql(undefined);
+ expect(enqueued[0].userData).to.be.eql(undefined);
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql(undefined);
+ expect(enqueued[1].userData).to.be.eql(undefined);
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql(undefined);
+ expect(enqueued[2].userData).to.be.eql(undefined);
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql(undefined);
+ expect(enqueued[3].userData).to.be.eql(undefined);
+ });
+
+ it('works with empty pseudoUrls[]', async () => {
+ const enqueued = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, []);
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql(undefined);
+ expect(enqueued[0].userData).to.be.eql(undefined);
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql(undefined);
+ expect(enqueued[1].userData).to.be.eql(undefined);
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql(undefined);
+ expect(enqueued[2].userData).to.be.eql(undefined);
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql(undefined);
+ expect(enqueued[3].userData).to.be.eql(undefined);
+ });
+
it('works with swapped pseudoUrls[] and requestQueue arguments', async () => {
const enqueued = [];
const queue = new RequestQueue('xxx');