Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add cheerio to enqueueLinks, add tests

Ondra Urban5efe45dc2f97
diff --git a/src/enqueue_links.js b/src/enqueue_links.js
index 535cf60fe..0facaab2d 100644
--- a/src/enqueue_links.js
+++ b/src/enqueue_links.js
@@ -41,6 +41,29 @@ export const constructPseudoUrlInstances = (pseudoUrls) => {
});
};
+/**
+ * Extracts URLs from a given Puppeteer Page.
+ *
+ * @param {Page} page
+ * @param {string} selector
+ * @returns {string[]}
+ */
+export const extractUrlsFromPage = async (page, selector) => {
+ /* istanbul ignore next */
+ return page.$$eval(selector, linkEls => linkEls.map(link => link.href).filter(href => !!href));
+};
+
+/**
+ * Extracts URLs from a given Cheerio object.
+ *
+ * @param {Function} $
+ * @param {string} selector
+ * @returns {string[]}
+ */
+export const extraxtUrlsFromCheerio = ($, selector) => {
+ return $(selector).map((i, el) => $(el).attr('href')).get().filter(href => !!href);
+};
+
/**
* Remove with 1.0.0
* @ignore
@@ -131,9 +154,9 @@ let logDeprecationWarning = true;
export const enqueueLinks = async (...args) => {
// TODO: Remove after v1.0.0 gets released.
// Refactor enqueueLinks to use an options object and keep backwards compatibility
- let page, selector, requestQueue, pseudoUrls, userData; // eslint-disable-line
+ let page, $, selector, requestQueue, pseudoUrls, userData; // eslint-disable-line
if (args.length === 1) {
- [{ page, selector = 'a', requestQueue, pseudoUrls, userData = {} }] = args;
+ [{ page, $, selector = 'a', requestQueue, pseudoUrls, userData = {} }] = args;
} else {
[page, selector = 'a', requestQueue, pseudoUrls, userData = {}] = args;
if (logDeprecationWarning) {
@@ -150,7 +173,14 @@ export const enqueueLinks = async (...args) => {
pseudoUrls = tmp;
}
- checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(page, 'page', 'Maybe Object');
+ checkParamOrThrow($, '$', 'Maybe Function');
+ if (!page && !$) {
+ throw new Error('One of the parameters "options.page" or "options.$" must be provided!');
+ }
+ if (page && $) {
+ throw new Error('Only one of the parameters "options.page" or "options.$" must be provided!');
+ }
checkParamOrThrow(selector, 'selector', 'String');
checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');
@@ -159,9 +189,7 @@ export const enqueueLinks = async (...args) => {
// Construct pseudoUrls from input where necessary.
const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);
- /* istanbul ignore next */
- const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
- const urls = await page.$$eval(selector, getHrefs);
+ const urls = page ? await extractUrlsFromPage(page, selector) : extraxtUrlsFromCheerio($, selector);
let requests = [];
if (pseudoUrlInstances.length) {
diff --git a/test/enqueue_links.js b/test/enqueue_links.js
index dcb6a7a63..d86283e2f 100644
--- a/test/enqueue_links.js
+++ b/test/enqueue_links.js
@@ -1,3 +1,4 @@
+import cheerio from 'cheerio';
import { expect } from 'chai';
import Apify from '../build/index';
import { enqueueLinks } from '../build/enqueue_links';
@@ -6,13 +7,24 @@ import { RequestQueue } from '../build/request_queue';
const { utils: { log } } = Apify;
describe('enqueueLinks()', () => {
- let browser;
- let page;
+ let ll;
+ before(() => {
+ ll = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ after(() => {
+ log.setLevel(ll);
+ });
+
+ describe('using Puppeteer', () => {
+ let browser;
+ let page;
- beforeEach(async () => {
- browser = await Apify.launchPuppeteer({ headless: true, dumpio: true });
- page = await browser.newPage();
- await page.setContent(`<html>
+ beforeEach(async () => {
+ browser = await Apify.launchPuppeteer({ headless: true });
+ page = await browser.newPage();
+ await page.setContent(`<html>
<head>
<title>Example</title>
</head>
@@ -32,262 +44,483 @@ describe('enqueueLinks()', () => {
</p>
</body>
</html>`);
- });
+ });
+
+ afterEach(async () => {
+ if (browser) await browser.close();
+ page = null;
+ browser = null;
+ });
+
+ it('works with PseudoUrl instances', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
- afterEach(async () => {
- if (browser) await browser.close();
- page = null;
- browser = null;
- });
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
- it('works with PseudoUrl instances', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
- ];
+ expect(enqueued).to.have.lengthOf(3);
- await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
- expect(enqueued).to.have.lengthOf(3);
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('POST');
- expect(enqueued[0].userData).to.be.eql({});
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
+ it('works with Actor UI output object', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ { purl: 'https://example.com/[(\\w|-|/)*]', method: 'POST' },
+ { purl: '[http|https]://cool.com/', userData: { foo: 'bar' } },
+ ];
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('POST');
- expect(enqueued[1].userData).to.be.eql({});
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData.foo).to.be.eql('bar');
- });
+ expect(enqueued).to.have.lengthOf(3);
- it('works with Actor UI output object', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- { purl: 'https://example.com/[(\\w|-|/)*]', method: 'POST' },
- { purl: '[http|https]://cool.com/', userData: { foo: 'bar' } },
- ];
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
- await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
- expect(enqueued).to.have.lengthOf(3);
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
+ it('works with string pseudoUrls', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ 'https://example.com/[(\\w|-|/)*]',
+ '[http|https]://cool.com/',
+ ];
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('POST');
- expect(enqueued[0].userData).to.be.eql({});
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('POST');
- expect(enqueued[1].userData).to.be.eql({});
+ expect(enqueued).to.have.lengthOf(3);
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData.foo).to.be.eql('bar');
- });
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
- it('works with string pseudoUrls', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- 'https://example.com/[(\\w|-|/)*]',
- '[http|https]://cool.com/',
- ];
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
- await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+ });
- expect(enqueued).to.have.lengthOf(3);
+ it('works with undefined pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
+ await enqueueLinks({ page, selector: '.click', requestQueue });
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
+ expect(enqueued).to.have.lengthOf(4);
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
- });
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
- it('works with undefined pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
- await enqueueLinks({ page, selector: '.click', requestQueue });
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
- expect(enqueued).to.have.lengthOf(4);
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
+ it('works with null pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: null });
- expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
+ expect(enqueued).to.have.lengthOf(4);
- expect(enqueued[3].url).to.be.eql('http://cool.com/');
- expect(enqueued[3].method).to.be.eql('GET');
- expect(enqueued[3].userData).to.be.eql({});
- });
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
- it('works with null pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
- await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: null });
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
- expect(enqueued).to.have.lengthOf(4);
+ it('works with empty pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: [] });
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
+ expect(enqueued).to.have.lengthOf(4);
- expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
- expect(enqueued[3].url).to.be.eql('http://cool.com/');
- expect(enqueued[3].method).to.be.eql('GET');
- expect(enqueued[3].userData).to.be.eql({});
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ // TODO Remove with 1.0.0
+ it('works with individual args instead of options object', async () => {
+ const enqueued = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const purls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ const originalLogWarning = log.warning;
+ const logOutput = [];
+ log.warning = (item) => { logOutput.push(item); };
+
+ try {
+ await enqueueLinks(page, '.click', purls, queue, { hello: 'world' });
+
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({ hello: 'world' });
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({ hello: 'world' });
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });
+ } finally {
+ log.warning = originalLogWarning;
+ }
+
+ expect(logOutput.length).to.be.eql(1);
+ expect(logOutput[0]).to.include('options object');
+ });
+ it('throws with sparse pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ null,
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ try {
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+ throw new Error('Wrong error.');
+ } catch (err) {
+ expect(err.message).to.include('pseudoUrls[1]');
+ expect(enqueued).to.have.lengthOf(0);
+ }
+ });
+
+ it('DEPRECATED: enqueueRequestsFromClickableElements()', async () => {
+ const enqueuedUrls = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = (request) => {
+ expect(request.method).to.be.eql('POST');
+ enqueuedUrls.push(request.url);
+
+ return Promise.resolve();
+ };
+ const purls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]'),
+ new Apify.PseudoUrl('[http|https]://cool.com/'),
+ ];
+
+ await Apify.utils.puppeteer.enqueueRequestsFromClickableElements(page, '.click', purls, queue, { method: 'POST' });
+
+ expect(enqueuedUrls).to.be.eql([
+ 'https://example.com/a/b/first',
+ 'https://example.com/a/b/third',
+ 'http://cool.com/',
+ ]);
+ });
});
- it('works with empty pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
+ describe('using Cheerio', () => {
+ let $;
+
+ beforeEach(async () => {
+ $ = cheerio.load(`<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+ </p>
+ <ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
+ <p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+ </p>
+ </body>
+ </html>`);
+ });
- await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: [] });
+ afterEach(async () => {
+ $ = null;
+ });
- expect(enqueued).to.have.lengthOf(4);
+ it('works with PseudoUrl instances', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
+ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
+ expect(enqueued).to.have.lengthOf(3);
- expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
- expect(enqueued[3].url).to.be.eql('http://cool.com/');
- expect(enqueued[3].method).to.be.eql('GET');
- expect(enqueued[3].userData).to.be.eql({});
- });
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
- // TODO Remove with 1.0.0
- it('works with individual args instead of options object', async () => {
- const enqueued = [];
- const queue = new RequestQueue('xxx');
- queue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const purls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
- ];
-
- const originalLogWarning = log.warning;
- const logOutput = [];
- log.warning = (item) => { logOutput.push(item); };
-
- try {
- await enqueueLinks(page, '.click', purls, queue, { hello: 'world' });
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
+ it('works with Actor UI output object', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ { purl: 'https://example.com/[(\\w|-|/)*]', method: 'POST' },
+ { purl: '[http|https]://cool.com/', userData: { foo: 'bar' } },
+ ];
+
+ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });
expect(enqueued).to.have.lengthOf(3);
expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
expect(enqueued[0].method).to.be.eql('POST');
- expect(enqueued[0].userData).to.be.eql({ hello: 'world' });
+ expect(enqueued[0].userData).to.be.eql({});
expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
expect(enqueued[1].method).to.be.eql('POST');
- expect(enqueued[1].userData).to.be.eql({ hello: 'world' });
+ expect(enqueued[1].userData).to.be.eql({});
expect(enqueued[2].url).to.be.eql('http://cool.com/');
expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });
- } finally {
- log.warning = originalLogWarning;
- }
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
+ it('works with string pseudoUrls', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ 'https://example.com/[(\\w|-|/)*]',
+ '[http|https]://cool.com/',
+ ];
+
+ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });
- expect(logOutput.length).to.be.eql(1);
- expect(logOutput[0]).to.include('options object');
- });
- it('throws with sparse pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
- null,
- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
- ];
-
- try {
- await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
- throw new Error('Wrong error.');
- } catch (err) {
- expect(err.message).to.include('pseudoUrls[1]');
- expect(enqueued).to.have.lengthOf(0);
- }
- });
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+ });
+
+ it('works with undefined pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
- it('DEPRECATED: enqueueRequestsFromClickableElements()', async () => {
- const enqueuedUrls = [];
- const queue = new RequestQueue('xxx');
- queue.addRequest = (request) => {
- expect(request.method).to.be.eql('POST');
- enqueuedUrls.push(request.url);
-
- return Promise.resolve();
- };
- const purls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]'),
- new Apify.PseudoUrl('[http|https]://cool.com/'),
- ];
-
- await Apify.utils.puppeteer.enqueueRequestsFromClickableElements(page, '.click', purls, queue, { method: 'POST' });
-
- expect(enqueuedUrls).to.be.eql([
- 'https://example.com/a/b/first',
- 'https://example.com/a/b/third',
- 'http://cool.com/',
- ]);
+ await enqueueLinks({ $, selector: '.click', requestQueue });
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ it('works with null pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls: null });
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ it('works with empty pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls: [] });
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ it('throws with sparse pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ null,
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ try {
+ await enqueueLinks({ $, selector: '.click', requestQueue, pseudoUrls });
+ throw new Error('Wrong error.');
+ } catch (err) {
+ expect(err.message).to.include('pseudoUrls[1]');
+ expect(enqueued).to.have.lengthOf(0);
+ }
+ });
});
});
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index feed17122..e18623a69 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -1,6 +1,7 @@
import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import Apify from '../build/index';
+import { RequestQueue } from '../build/request_queue';
const { utils: { log } } = Apify;