Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Move enqueueLinks() to its own file

Ondra Urban0b112a80f569
diff --git a/src/enqueue_links.js b/src/enqueue_links.js
new file mode 100644
index 000000000..535cf60fe
--- /dev/null
+++ b/src/enqueue_links.js
@@ -0,0 +1,184 @@
+import _ from 'underscore';
+import log from 'apify-shared/log';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
+import { RequestQueue, RequestQueueLocal } from './request_queue';
+import Request from './request';
+import PseudoUrl from './pseudo_url';
+
+/**
+ * To enable direct use of the Actor UI `pseudoUrls` output while keeping high performance,
+ * all the pseudoUrls from the output are only constructed once and kept in a cache
+ * by the `enqueueLinks()` function.
+ * @ignore
+ */
+const enqueueLinksCache = new Map();
+export const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
+
+/**
+ * Helper factory used in the `enqeueLinks()` function.
+ * @param {Array} pseudoUrls
+ * @returns {Array}
+ * @ignore
+ */
+export const constructPseudoUrlInstances = (pseudoUrls) => {
+ return pseudoUrls.map((item, idx) => {
+ // Get pseudoUrl instance from cache.
+ let pUrl = enqueueLinksCache.get(item);
+ if (pUrl) return pUrl;
+ // Nothing in cache, make a new instance.
+ checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'Object|String');
+ if (item instanceof PseudoUrl) pUrl = item;
+ else if (typeof item === 'string') pUrl = new PseudoUrl(item);
+ else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));
+ // Manage cache
+ enqueueLinksCache.set(item, pUrl);
+ if (enqueueLinksCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
+ const key = enqueueLinksCache.keys().next().value;
+ enqueueLinksCache.delete(key);
+ }
+ return pUrl;
+ });
+};
+
+/**
+ * Remove with 1.0.0
+ * @ignore
+ * @todo deprecate
+ */
+let logDeprecationWarning = true;
+
+/**
+ * The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default)
+ * either in a Puppeteer page, or in a Cheerio object (parsed HTML),
+ * and enqueues the corresponding links to the provided {@link RequestQueue}.
+ * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
+ * and override settings of the enqueued {@link Request} objects.
+ *
+ * *IMPORTANT*: This is a work in progress. Currently the function only supports elements with
+ * `href` attribute pointing to a URL. However, in the future the function will also support
+ * JavaScript links, buttons and form submissions when used with a Puppeteer Page.
+ *
+ * **Example usage**
+ *
+ * ```javascript
+ * const Apify = require('apify');
+ *
+ * const browser = await Apify.launchPuppeteer();
+ * const page = await browser.goto('https://www.example.com');
+ * const requestQueue = await Apify.openRequestQueue();
+ *
+ * await Apify.utils.puppeteer.enqueueLinks({
+ * page,
+ * requestQueue,
+ * selector: 'a.product-detail',
+ * pseudoUrls: [
+ * 'https://www.example.com/handbags/[.*]'
+ * 'https://www.example.com/purses/[.*]'
+ * ],
+ * });
+ * ```
+ *
+ * @param {Object} options
+ * @param {Page} options.page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
+ * Either `page` or `$` option must be provided.
+ * @param {Cheerio} options.$
+ * <a href="https://www.npmjs.com/package/cheerio" target="_blank"><code>Cheerio</code></a> object.
+ * Either `page` or `$` option must be provided.
+ * @param {RequestQueue} options.requestQueue
+ * A request queue to which the URLs will be enqueued.
+ * @param {String} [options.selector='a']
+ * A CSS selector matching links to be enqueued.
+ * @param {PseudoUrl[]|Object[]|String[]} [options.pseudoUrls]
+ * An array of {@link PseudoUrl}s matching the URLs to be enqueued,
+ * or an array of strings or objects from which the {@link PseudoUrl}s can be constructed.
+ * The objects must include at least the `purl` property, which holds the pseudo-URL string.
+ * All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor.
+ * which lets you specify special properties for the enqueued {@link Request} objects.
+ *
+ * If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
+ * enqueues all links found on the page.
+ * @param {Object} [options.userData]
+ * An object that will be merged with the new {@link Request}'s `userData`, overriding any values that
+ * were set via templating from `pseudoUrls`. This is useful when you need to override generic
+ * `userData` set by the {@link PseudoUrl} template in specific use cases.
+ * **Example:**
+ * ```
+ * // pseudoUrl.userData
+ * {
+ * name: 'John',
+ * surname: 'Doe',
+ * }
+ *
+ * // userData
+ * {
+ * name: 'Albert',
+ * age: 31
+ * }
+ *
+ * // enqueued request.userData
+ * {
+ * name: 'Albert',
+ * surname: 'Doe',
+ * age: 31,
+ * }
+ * ```
+ * @return {Promise<QueueOperationInfo[]>}
+ * Promise that resolves to an array of {@link QueueOperationInfo} objects.
+ * @memberOf puppeteer
+ */
+export const enqueueLinks = async (...args) => {
+ // TODO: Remove after v1.0.0 gets released.
+ // Refactor enqueueLinks to use an options object and keep backwards compatibility
+ let page, selector, requestQueue, pseudoUrls, userData; // eslint-disable-line
+ if (args.length === 1) {
+ [{ page, selector = 'a', requestQueue, pseudoUrls, userData = {} }] = args;
+ } else {
+ [page, selector = 'a', requestQueue, pseudoUrls, userData = {}] = args;
+ if (logDeprecationWarning) {
+ log.warning('Passing individual arguments to enqueueLinks() is deprecated. '
+ + 'Use an options object: enqueueLinks({ page, selector, requestQueue, pseudoUrls, userData }) instead.');
+ logDeprecationWarning = false;
+ }
+ }
+
+ // Check for pseudoUrls as a third parameter.
+ if (Array.isArray(requestQueue)) {
+ const tmp = requestQueue;
+ requestQueue = pseudoUrls;
+ pseudoUrls = tmp;
+ }
+
+ checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(selector, 'selector', 'String');
+ checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
+ checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');
+ checkParamOrThrow(userData, 'userData', 'Maybe Object');
+
+ // Construct pseudoUrls from input where necessary.
+ const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);
+
+ /* istanbul ignore next */
+ const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
+ const urls = await page.$$eval(selector, getHrefs);
+ let requests = [];
+
+ if (pseudoUrlInstances.length) {
+ urls.forEach((url) => {
+ pseudoUrlInstances
+ .filter(purl => purl.matches(url))
+ .forEach(purl => requests.push(purl.createRequest(url)));
+ });
+ } else {
+ requests = urls.map(url => new Request({ url }));
+ }
+
+ const queueOperationInfos = [];
+ for (const request of requests) {
+ // Inject custom userData
+ Object.assign(request.userData, userData);
+ queueOperationInfos.push(await requestQueue.addRequest(request));
+ }
+ return queueOperationInfos;
+};
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 87d67878d..b91d2f877 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -7,7 +7,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import { RequestQueue, RequestQueueLocal } from './request_queue';
import Request from './request';
-import PseudoUrl from './pseudo_url';
+import { enqueueLinks } from './enqueue_links';
const jqueryPath = require.resolve('jquery');
const underscorePath = require.resolve('underscore');
@@ -170,177 +170,6 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque
return queueOperationInfos;
};
-/**
- * To enable direct use of the Actor UI `pseudoUrls` output while keeping high performance,
- * all the pseudoUrls from the output are only constructed once and kept in a cache
- * by the `enqueueLinks()` function.
- * @ignore
- */
-const enqueueLinksCache = new Map();
-export const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
-
-/**
- * Helper factory used in the `enqeueLinks()` function.
- * @param {Array} pseudoUrls
- * @returns {Array}
- * @ignore
- */
-export const constructPseudoUrlInstances = (pseudoUrls) => {
- return pseudoUrls.map((item, idx) => {
- // Get pseudoUrl instance from cache.
- let pUrl = enqueueLinksCache.get(item);
- if (pUrl) return pUrl;
- // Nothing in cache, make a new instance.
- checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'Object|String');
- if (item instanceof PseudoUrl) pUrl = item;
- else if (typeof item === 'string') pUrl = new PseudoUrl(item);
- else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));
- // Manage cache
- enqueueLinksCache.set(item, pUrl);
- if (enqueueLinksCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
- const key = enqueueLinksCache.keys().next().value;
- enqueueLinksCache.delete(key);
- }
- return pUrl;
- });
-};
-
-/**
- * Remove with 1.0.0
- * @ignore
- * @todo deprecate
- */
-let logDeprecationWarning = true;
-
-/**
- * The function finds HTML anchor (`&lt;a&gt;`) elements matching a specific CSS selector on a Puppeteer page,
- * and enqueues the corresponding links to the provided {@link RequestQueue}.
- * Optionally, the function allows you to filter the target links URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
- *
- * *IMPORTANT*: This is a work in progress. Currently the function only supports `&lt;a&gt;` elements with
- * `href` attribute point to some URL. However, in the future the function will also support
- * JavaScript links, buttons and form submissions.
- *
- * **Example usage**
- *
- * ```javascript
- * const Apify = require('apify');
- *
- * const browser = await Apify.launchPuppeteer();
- * const page = await browser.goto('https://www.example.com');
- * const requestQueue = await Apify.openRequestQueue();
- *
- * await Apify.utils.puppeteer.enqueueLinks({
- * page,
- * requestQueue,
- * selector: 'a.product-detail',
- * pseudoUrls: [
- * 'https://www.example.com/handbags/[.*]'
- * 'https://www.example.com/purses/[.*]'
- * ],
- * });
- * ```
- *
- * @param {Object} options
- * @param {Page} options.page
- * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
- * @param {RequestQueue} options.requestQueue
- * A request queue to which the URLs will be enqueued.
- * @param {String} [options.selector='a']
- * A CSS selector matching links to be enqueued.
- * @param {PseudoUrl[]|Object[]|String[]} [options.pseudoUrls]
- * An array of {@link PseudoUrl}s matching the URLs to be enqueued,
- * or an array of strings or objects from which the {@link PseudoUrl}s can be constructed.
- * The objects must include at least the `purl` property, which holds the pseudo-URL string.
- * All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor.
- * which lets you specify special properties for the enqueued {@link Request} objects.
- *
- * If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
- * enqueues all links found on the page.
- * @param {Object} [options.userData]
- * An object that will be merged with the new {@link Request}'s `userData`, overriding any values that
- * were set via templating from `pseudoUrls`. This is useful when you need to override generic
- * `userData` set by the {@link PseudoUrl} template in specific use cases.
- * **Example:**
- * ```
- * // pseudoUrl.userData
- * {
- * name: 'John',
- * surname: 'Doe',
- * }
- *
- * // userData
- * {
- * name: 'Albert',
- * age: 31
- * }
- *
- * // enqueued request.userData
- * {
- * name: 'Albert',
- * surname: 'Doe',
- * age: 31,
- * }
- * ```
- * @return {Promise<QueueOperationInfo[]>}
- * Promise that resolves to an array of {@link QueueOperationInfo} objects.
- * @memberOf puppeteer
- */
-const enqueueLinks = async (...args) => {
- // TODO: Remove after v1.0.0 gets released.
- // Refactor enqueueLinks to use an options object and keep backwards compatibility
- let page, selector, requestQueue, pseudoUrls, userData; // eslint-disable-line
- if (args.length === 1) {
- [{ page, selector = 'a', requestQueue, pseudoUrls, userData = {} }] = args;
- } else {
- [page, selector = 'a', requestQueue, pseudoUrls, userData = {}] = args;
- if (logDeprecationWarning) {
- log.warning('Passing individual arguments to enqueueLinks() is deprecated. '
- + 'Use an options object: enqueueLinks({ page, selector, requestQueue, pseudoUrls, userData }) instead.');
- logDeprecationWarning = false;
- }
- }
-
- // Check for pseudoUrls as a third parameter.
- if (Array.isArray(requestQueue)) {
- const tmp = requestQueue;
- requestQueue = pseudoUrls;
- pseudoUrls = tmp;
- }
-
- checkParamOrThrow(page, 'page', 'Object');
- checkParamOrThrow(selector, 'selector', 'String');
- checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');
- checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');
- checkParamOrThrow(userData, 'userData', 'Maybe Object');
-
- // Construct pseudoUrls from input where necessary.
- const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);
-
- /* istanbul ignore next */
- const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);
- const urls = await page.$$eval(selector, getHrefs);
- let requests = [];
-
- if (pseudoUrlInstances.length) {
- urls.forEach((url) => {
- pseudoUrlInstances
- .filter(purl => purl.matches(url))
- .forEach(purl => requests.push(purl.createRequest(url)));
- });
- } else {
- requests = urls.map(url => new Request({ url }));
- }
-
- const queueOperationInfos = [];
- for (const request of requests) {
- // Inject custom userData
- Object.assign(request.userData, userData);
- queueOperationInfos.push(await requestQueue.addRequest(request));
- }
- return queueOperationInfos;
-};
/**
* Forces the Puppeteer browser tab to block loading certain HTTP resources.
diff --git a/test/enqueue_links.js b/test/enqueue_links.js
new file mode 100644
index 000000000..dcb6a7a63
--- /dev/null
+++ b/test/enqueue_links.js
@@ -0,0 +1,293 @@
+import { expect } from 'chai';
+import Apify from '../build/index';
+import { enqueueLinks } from '../build/enqueue_links';
+import { RequestQueue } from '../build/request_queue';
+
+const { utils: { log } } = Apify;
+
+describe('enqueueLinks()', () => {
+ let browser;
+ let page;
+
+ beforeEach(async () => {
+ browser = await Apify.launchPuppeteer({ headless: true, dumpio: true });
+ page = await browser.newPage();
+ await page.setContent(`<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+ </p>
+ <ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
+ <p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+ </p>
+ </body>
+ </html>`);
+ });
+
+ afterEach(async () => {
+ if (browser) await browser.close();
+ page = null;
+ browser = null;
+ });
+
+ it('works with PseudoUrl instances', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
+ it('works with Actor UI output object', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ { purl: 'https://example.com/[(\\w|-|/)*]', method: 'POST' },
+ { purl: '[http|https]://cool.com/', userData: { foo: 'bar' } },
+ ];
+
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData.foo).to.be.eql('bar');
+ });
+
+ it('works with string pseudoUrls', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ 'https://example.com/[(\\w|-|/)*]',
+ '[http|https]://cool.com/',
+ ];
+
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+ });
+
+ it('works with undefined pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ page, selector: '.click', requestQueue });
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ it('works with null pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: null });
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ it('works with empty pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: [] });
+
+ expect(enqueued).to.have.lengthOf(4);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('GET');
+ expect(enqueued[0].userData).to.be.eql({});
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('GET');
+ expect(enqueued[1].userData).to.be.eql({});
+
+ expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({});
+
+ expect(enqueued[3].url).to.be.eql('http://cool.com/');
+ expect(enqueued[3].method).to.be.eql('GET');
+ expect(enqueued[3].userData).to.be.eql({});
+ });
+
+ // TODO Remove with 1.0.0
+ it('works with individual args instead of options object', async () => {
+ const enqueued = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const purls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ const originalLogWarning = log.warning;
+ const logOutput = [];
+ log.warning = (item) => { logOutput.push(item); };
+
+ try {
+ await enqueueLinks(page, '.click', purls, queue, { hello: 'world' });
+
+ expect(enqueued).to.have.lengthOf(3);
+
+ expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
+ expect(enqueued[0].method).to.be.eql('POST');
+ expect(enqueued[0].userData).to.be.eql({ hello: 'world' });
+
+ expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
+ expect(enqueued[1].method).to.be.eql('POST');
+ expect(enqueued[1].userData).to.be.eql({ hello: 'world' });
+
+ expect(enqueued[2].url).to.be.eql('http://cool.com/');
+ expect(enqueued[2].method).to.be.eql('GET');
+ expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });
+ } finally {
+ log.warning = originalLogWarning;
+ }
+
+ expect(logOutput.length).to.be.eql(1);
+ expect(logOutput[0]).to.include('options object');
+ });
+ it('throws with sparse pseudoUrls[]', async () => {
+ const enqueued = [];
+ const requestQueue = new RequestQueue('xxx');
+ requestQueue.addRequest = async (request) => {
+ enqueued.push(request);
+ };
+ const pseudoUrls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
+ null,
+ new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
+ ];
+
+ try {
+ await enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
+ throw new Error('Wrong error.');
+ } catch (err) {
+ expect(err.message).to.include('pseudoUrls[1]');
+ expect(enqueued).to.have.lengthOf(0);
+ }
+ });
+
+ it('DEPRECATED: enqueueRequestsFromClickableElements()', async () => {
+ const enqueuedUrls = [];
+ const queue = new RequestQueue('xxx');
+ queue.addRequest = (request) => {
+ expect(request.method).to.be.eql('POST');
+ enqueuedUrls.push(request.url);
+
+ return Promise.resolve();
+ };
+ const purls = [
+ new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]'),
+ new Apify.PseudoUrl('[http|https]://cool.com/'),
+ ];
+
+ await Apify.utils.puppeteer.enqueueRequestsFromClickableElements(page, '.click', purls, queue, { method: 'POST' });
+
+ expect(enqueuedUrls).to.be.eql([
+ 'https://example.com/a/b/first',
+ 'https://example.com/a/b/third',
+ 'http://cool.com/',
+ ]);
+ });
+});
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index b8cae73f9..feed17122 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -1,7 +1,6 @@
import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import Apify from '../build/index';
-import { RequestQueue } from '../build/request_queue';
const { utils: { log } } = Apify;
@@ -99,291 +98,8 @@ describe('Apify.utils.puppeteer', () => {
}
});
- describe('enqueueLinks()', () => {
- let browser;
- let page;
-
- beforeEach(async () => {
- browser = await Apify.launchPuppeteer({ headless: true, dumpio: true });
- page = await browser.newPage();
- await page.setContent(`<html>
- <head>
- <title>Example</title>
- </head>
- <body>
- <p>
- The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
- </p>
- <ul>
- <li>These aren't the Droids you're looking for</li>
- <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
- <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
- </ul>
- <a class="click" href="https://another.com/a/fifth">The Greatest Science Fiction Quotes Of All Time</a>
- <p>
- Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
- just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
- </p>
- </body>
- </html>`);
- });
-
- afterEach(async () => {
- if (browser) await browser.close();
- page = null;
- browser = null;
- });
-
- it('works with PseudoUrl instances', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
- ];
-
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
-
- expect(enqueued).to.have.lengthOf(3);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('POST');
- expect(enqueued[0].userData).to.be.eql({});
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('POST');
- expect(enqueued[1].userData).to.be.eql({});
-
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData.foo).to.be.eql('bar');
- });
-
- it('works with Actor UI output object', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- { purl: 'https://example.com/[(\\w|-|/)*]', method: 'POST' },
- { purl: '[http|https]://cool.com/', userData: { foo: 'bar' } },
- ];
-
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
-
- expect(enqueued).to.have.lengthOf(3);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('POST');
- expect(enqueued[0].userData).to.be.eql({});
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('POST');
- expect(enqueued[1].userData).to.be.eql({});
-
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData.foo).to.be.eql('bar');
- });
-
- it('works with string pseudoUrls', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- 'https://example.com/[(\\w|-|/)*]',
- '[http|https]://cool.com/',
- ];
-
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
-
- expect(enqueued).to.have.lengthOf(3);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
-
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
- });
-
- it('works with undefined pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
-
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue });
-
- expect(enqueued).to.have.lengthOf(4);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
-
- expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
-
- expect(enqueued[3].url).to.be.eql('http://cool.com/');
- expect(enqueued[3].method).to.be.eql('GET');
- expect(enqueued[3].userData).to.be.eql({});
- });
-
- it('works with null pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
-
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: null });
-
- expect(enqueued).to.have.lengthOf(4);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
-
- expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
-
- expect(enqueued[3].url).to.be.eql('http://cool.com/');
- expect(enqueued[3].method).to.be.eql('GET');
- expect(enqueued[3].userData).to.be.eql({});
- });
-
- it('works with empty pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
-
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: [] });
-
- expect(enqueued).to.have.lengthOf(4);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('GET');
- expect(enqueued[0].userData).to.be.eql({});
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('GET');
- expect(enqueued[1].userData).to.be.eql({});
-
- expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({});
-
- expect(enqueued[3].url).to.be.eql('http://cool.com/');
- expect(enqueued[3].method).to.be.eql('GET');
- expect(enqueued[3].userData).to.be.eql({});
- });
-
- // TODO Remove with 1.0.0
- it('works with individual args instead of options object', async () => {
- const enqueued = [];
- const queue = new RequestQueue('xxx');
- queue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const purls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
- ];
-
- const originalLogWarning = log.warning;
- const logOutput = [];
- log.warning = (item) => { logOutput.push(item); };
-
- try {
- await Apify.utils.puppeteer.enqueueLinks(page, '.click', purls, queue, { hello: 'world' });
-
- expect(enqueued).to.have.lengthOf(3);
-
- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
- expect(enqueued[0].method).to.be.eql('POST');
- expect(enqueued[0].userData).to.be.eql({ hello: 'world' });
-
- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');
- expect(enqueued[1].method).to.be.eql('POST');
- expect(enqueued[1].userData).to.be.eql({ hello: 'world' });
-
- expect(enqueued[2].url).to.be.eql('http://cool.com/');
- expect(enqueued[2].method).to.be.eql('GET');
- expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });
- } finally {
- log.warning = originalLogWarning;
- }
-
- expect(logOutput.length).to.be.eql(1);
- expect(logOutput[0]).to.include('options object');
- });
- it('throws with sparse pseudoUrls[]', async () => {
- const enqueued = [];
- const requestQueue = new RequestQueue('xxx');
- requestQueue.addRequest = async (request) => {
- enqueued.push(request);
- };
- const pseudoUrls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),
- null,
- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),
- ];
-
- try {
- await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });
- throw new Error('Wrong error.');
- } catch (err) {
- expect(err.message).to.include('pseudoUrls[1]');
- expect(enqueued).to.have.lengthOf(0);
- }
- });
-
- it('DEPRECATED: enqueueRequestsFromClickableElements()', async () => {
- const enqueuedUrls = [];
- const queue = new RequestQueue('xxx');
- queue.addRequest = (request) => {
- expect(request.method).to.be.eql('POST');
- enqueuedUrls.push(request.url);
-
- return Promise.resolve();
- };
- const purls = [
- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]'),
- new Apify.PseudoUrl('[http|https]://cool.com/'),
- ];
-
- await Apify.utils.puppeteer.enqueueRequestsFromClickableElements(page, '.click', purls, queue, { method: 'POST' });
-
- expect(enqueuedUrls).to.be.eql([
- 'https://example.com/a/b/first',
- 'https://example.com/a/b/third',
- 'http://cool.com/',
- ]);
- });
+ it('enqueueLinks() exists in this namespace', async () => {
+ expect(Apify.utils.puppeteer.enqueueLinks).to.be.a('function');
});
it('supports blockResources() with default values', async () => {