Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add userData to enqueueLinks
Ondra Urban18e136d1c391
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 65c08c32f..a09044cb3 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -197,33 +197,66 @@ let logDeprecationWarning = true;** *WARNING*: This is work in progress. Currently the function doesn't click elements and only takes their `href` attribute and so* is working only for link (`a`) elements, but not for buttons or JavaScript links.- *- * @param {Page} page+ * @param {Object} options+ * @param {Page} options.page* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.- * @param {String} selector- * CSS selector matching elements to be clicked.- * @param {RequestQueue} requestQueue+ * @param {RequestQueue} options.requestQueue* {@link RequestQueue} instance where URLs will be enqueued.- * @param {PseudoUrl[]|Object[]|String[]} [pseudoUrls]+ * @param {String} [options.selector='a']+ * CSS selector matching elements to be clicked.+ * @param {PseudoUrl[]|Object[]|String[]} [options.pseudoUrls]* An array of {@link PseudoUrl}s matching the URLs to be enqueued,* or an array of Strings or Objects from which the {@link PseudoUrl}s should be constructed* The Objects must include at least a `purl` property, which holds a pseudoUrl string.* All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor.* If `pseudoUrls` is an empty array, null or undefined, then the function* enqueues all links found on the page.+ * @param {Object} [options.userData]+ * Object that will be merged with the new Request's userData, overriding any values that+ * were set via templating from pseudoUrls. This is useful when you need to override generic+ * userData set by PseudoURL template in specific use cases.+ * **Example:**+ * ```+ * // pseudoUrl.userData+ * {+ * name: 'John',+ * surname: 'Doe',+ * }+ *+ * // userData+ * {+ * name: 'Albert',+ * age: 31+ * }+ *+ * // enqueued request.userData+ * {+ * name: 'Albert',+ * surname: 'Doe',+ * age: 31,+ * }+ * ```* @return {Promise<QueueOperationInfo[]>}* Promise that resolves to an array of {@link QueueOperationInfo} objects.* @memberOf puppeteer*/-const enqueueLinks = async (page, selector, requestQueue, pseudoUrls) => {+const enqueueLinks = async (...args) => {// TODO: Remove after v1.0.0 gets released.- // Check for pseudoUrls as a third parameter.- if (Array.isArray(requestQueue)) {+ // Refactor enqueueLinks to use an options object and keep backwards compatibility+ let page, selector, requestQueue, pseudoUrls, userData; // eslint-disable-line+ if (args.length === 1) {+ [{ page, selector = 'a', requestQueue, pseudoUrls, userData = {} }] = args;+ } else {+ [page, selector = 'ą', requestQueue, pseudoUrls, userData = {}] = args;if (logDeprecationWarning) {- log.warning('Argument "pseudoUrls" as the third parameter to enqueueLinks() is deprecated. '- + 'Use enqueueLinks(page, selector, requestQueue, pseudoUrls) instead. "pseudoUrls" are now optional.');+ log.warning('Passing individual arguments to enqueueLinks() is deprecated. '+ + 'Use an options object: enqueueLinks({ page, selector, requestQueue, pseudoUrls, userData }) instead.');logDeprecationWarning = false;}+ }++ // Check for pseudoUrls as a third parameter.+ if (Array.isArray(requestQueue)) {const tmp = requestQueue;requestQueue = pseudoUrls;pseudoUrls = tmp;@@ -233,6 +266,7 @@ const enqueueLinks = async (page, selector, requestQueue, pseudoUrls) => {checkParamOrThrow(selector, 'selector', 'String');checkParamPrototypeOrThrow(requestQueue, 'requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue');checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');+ checkParamOrThrow(userData, 'userData', 'Object');// Construct pseudoUrls from input where necessary.const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);@@ -249,11 +283,13 @@ const enqueueLinks = async (page, selector, requestQueue, pseudoUrls) => {.forEach(purl => requests.push(purl.createRequest(url)));});} else {- requests = urls.map(url => ({ url }));+ requests = urls.map(url => new Request({ url }));}const queueOperationInfos = [];for (const request of requests) {+ // Inject custom userData+ Object.assign(request.userData, userData);queueOperationInfos.push(await requestQueue.addRequest(request));}return queueOperationInfos;diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex f8ff0af10..b8cae73f9 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -136,16 +136,16 @@ describe('Apify.utils.puppeteer', () => {it('works with PseudoUrl instances', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- const purls = [+ const pseudoUrls = [new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),];- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, purls);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });expect(enqueued).to.have.lengthOf(3);@@ -164,16 +164,16 @@ describe('Apify.utils.puppeteer', () => {it('works with Actor UI output object', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- const purls = [+ const pseudoUrls = [{ purl: 'https://example.com/[(\\w|-|/)*]', method: 'POST' },{ purl: '[http|https]://cool.com/', userData: { foo: 'bar' } },];- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, purls);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });expect(enqueued).to.have.lengthOf(3);@@ -192,16 +192,16 @@ describe('Apify.utils.puppeteer', () => {it('works with string pseudoUrls', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- const purls = [+ const pseudoUrls = ['https://example.com/[(\\w|-|/)*]','[http|https]://cool.com/',];- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, purls);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });expect(enqueued).to.have.lengthOf(3);@@ -220,89 +220,90 @@ describe('Apify.utils.puppeteer', () => {it('works with undefined pseudoUrls[]', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue });expect(enqueued).to.have.lengthOf(4);expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');- expect(enqueued[0].method).to.be.eql(undefined);- expect(enqueued[0].userData).to.be.eql(undefined);+ expect(enqueued[0].method).to.be.eql('GET');+ expect(enqueued[0].userData).to.be.eql({});expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');- expect(enqueued[1].method).to.be.eql(undefined);- expect(enqueued[1].userData).to.be.eql(undefined);+ expect(enqueued[1].method).to.be.eql('GET');+ expect(enqueued[1].userData).to.be.eql({});expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');- expect(enqueued[2].method).to.be.eql(undefined);- expect(enqueued[2].userData).to.be.eql(undefined);+ expect(enqueued[2].method).to.be.eql('GET');+ expect(enqueued[2].userData).to.be.eql({});expect(enqueued[3].url).to.be.eql('http://cool.com/');- expect(enqueued[3].method).to.be.eql(undefined);- expect(enqueued[3].userData).to.be.eql(undefined);+ expect(enqueued[3].method).to.be.eql('GET');+ expect(enqueued[3].userData).to.be.eql({});});it('works with null pseudoUrls[]', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, null);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: null });expect(enqueued).to.have.lengthOf(4);expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');- expect(enqueued[0].method).to.be.eql(undefined);- expect(enqueued[0].userData).to.be.eql(undefined);+ expect(enqueued[0].method).to.be.eql('GET');+ expect(enqueued[0].userData).to.be.eql({});expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');- expect(enqueued[1].method).to.be.eql(undefined);- expect(enqueued[1].userData).to.be.eql(undefined);+ expect(enqueued[1].method).to.be.eql('GET');+ expect(enqueued[1].userData).to.be.eql({});expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');- expect(enqueued[2].method).to.be.eql(undefined);- expect(enqueued[2].userData).to.be.eql(undefined);+ expect(enqueued[2].method).to.be.eql('GET');+ expect(enqueued[2].userData).to.be.eql({});expect(enqueued[3].url).to.be.eql('http://cool.com/');- expect(enqueued[3].method).to.be.eql(undefined);- expect(enqueued[3].userData).to.be.eql(undefined);+ expect(enqueued[3].method).to.be.eql('GET');+ expect(enqueued[3].userData).to.be.eql({});});it('works with empty pseudoUrls[]', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, []);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls: [] });expect(enqueued).to.have.lengthOf(4);expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');- expect(enqueued[0].method).to.be.eql(undefined);- expect(enqueued[0].userData).to.be.eql(undefined);+ expect(enqueued[0].method).to.be.eql('GET');+ expect(enqueued[0].userData).to.be.eql({});expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');- expect(enqueued[1].method).to.be.eql(undefined);- expect(enqueued[1].userData).to.be.eql(undefined);+ expect(enqueued[1].method).to.be.eql('GET');+ expect(enqueued[1].userData).to.be.eql({});expect(enqueued[2].url).to.be.eql('https://another.com/a/fifth');- expect(enqueued[2].method).to.be.eql(undefined);- expect(enqueued[2].userData).to.be.eql(undefined);+ expect(enqueued[2].method).to.be.eql('GET');+ expect(enqueued[2].userData).to.be.eql({});expect(enqueued[3].url).to.be.eql('http://cool.com/');- expect(enqueued[3].method).to.be.eql(undefined);- expect(enqueued[3].userData).to.be.eql(undefined);+ expect(enqueued[3].method).to.be.eql('GET');+ expect(enqueued[3].userData).to.be.eql({});});- it('works with swapped pseudoUrls[] and requestQueue arguments', async () => {+ // TODO Remove with 1.0.0+ it('works with individual args instead of options object', async () => {const enqueued = [];const queue = new RequestQueue('xxx');queue.addRequest = async (request) => {@@ -318,42 +319,42 @@ describe('Apify.utils.puppeteer', () => {log.warning = (item) => { logOutput.push(item); };try {- await Apify.utils.puppeteer.enqueueLinks(page, '.click', purls, queue);+ await Apify.utils.puppeteer.enqueueLinks(page, '.click', purls, queue, { hello: 'world' });expect(enqueued).to.have.lengthOf(3);expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');expect(enqueued[0].method).to.be.eql('POST');- expect(enqueued[0].userData).to.be.eql({});+ expect(enqueued[0].userData).to.be.eql({ hello: 'world' });expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');expect(enqueued[1].method).to.be.eql('POST');- expect(enqueued[1].userData).to.be.eql({});+ expect(enqueued[1].userData).to.be.eql({ hello: 'world' });expect(enqueued[2].url).to.be.eql('http://cool.com/');expect(enqueued[2].method).to.be.eql('GET');- expect(enqueued[2].userData.foo).to.be.eql('bar');+ expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });} finally {log.warning = originalLogWarning;}expect(logOutput.length).to.be.eql(1);- expect(logOutput[0]).to.include('Argument "pseudoUrls"');+ expect(logOutput[0]).to.include('options object');});it('throws with sparse pseudoUrls[]', async () => {const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.addRequest = async (request) => {enqueued.push(request);};- const purls = [+ const pseudoUrls = [new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),null,new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),];try {- await Apify.utils.puppeteer.enqueueLinks(page, '.click', queue, purls);+ await Apify.utils.puppeteer.enqueueLinks({ page, selector: '.click', requestQueue, pseudoUrls });throw new Error('Wrong error.');} catch (err) {expect(err.message).to.include('pseudoUrls[1]');