Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add transformRequestFunction to enqueueLinks
Ondra Urban1221f813b2e5
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fb14ccf9a..574e41587 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,8 +4,8 @@ NEXTIf you wish to continue using Web Driver, please stay on Apify SDK version ^0.14.15- **BREAKING CHANGE**: `Dataset.getData()` throws an error if user provides an unsupported optionwhen using local disk storage.-- **BREAKING CHANGE**: `Requests` with `method` other than `GET` will now have their `uniqueKey` computed- from a combination of `url`, `method` and `payload`.+- **DEPRECATED**: `options.userData` of `Apify.utils.enqueueLinks()` is deprecated.+ Use `options.transformRequestFunction` instead.- Improve logging of memory overload errors.- Improve error message in `Apify.call()`.- Fix multiple log lines appearing when a crawler was about to finish.diff --git a/docs/api/puppeteer.md b/docs/api/puppeteer.mdindex df2ed7841..c1295e6f7 100644--- a/docs/api/puppeteer.md+++ b/docs/api/puppeteer.md@@ -127,12 +127,13 @@ await Apify.utils.enqueueLinksByClickingElements({<td><code>[options.transformRequestFunction]</code></td><td><code>function</code></td><td></td></tr><tr>-<td colspan="3"><p>Just before a new <a href="request"><code>Request</code></a> is enqueued to the <a href="requestqueue"><code>RequestQueue</code></a>, this function can be used to remove it- or modify its contents such as <code>userData</code>, <code>payload</code> or, most importantly <code>uniqueKey</code>. This is useful when you need- to enqueue multiple <code>Requests</code> to the queue that share the same URL, but differ in methods or payloads,+<td colspan="3"><p>Just before a new <a href="request"><code>Request</code></a> is constructed and enqueued to the <a href="requestqueue"><code>RequestQueue</code></a>, this function can be used+ to remove it or modify its contents such as <code>userData</code>, <code>payload</code> or, most importantly <code>uniqueKey</code>. This is useful+ when you need to enqueue multiple <code>Requests</code> to the queue that share the same URL, but differ in methods or payloads,or to dynamically update or create <code>userData</code>.</p>-<p> For example: by generating your own <code>uniqueKey</code> from a combination of <code>url</code>, <code>method</code> and <code>payload</code> you enable crawling- of websites that navigate using form submits (POST requests).</p>+<p> For example: by adding <code>useExtendedUniqueKey: true</code> to the <code>request</code> object, <code>uniqueKey</code> will be computed from+ a combination of <code>url</code>, <code>method</code> and <code>payload</code> which enables crawling of websites that navigate using form submits+ (POST requests).</p></td></tr><tr><td><code>[options.waitForPageIdleSecs]</code></td><td><code>number</code></td><td><code>1</code></td></tr>diff --git a/docs/api/utils.md b/docs/api/utils.mdindex 61c93a2a8..bf136ba22 100644--- a/docs/api/utils.md+++ b/docs/api/utils.md@@ -117,27 +117,16 @@ await Apify.utils.enqueueLinks({<p> If <code>pseudoUrls</code> is an empty array, <code>null</code> or <code>undefined</code>, then the functionenqueues all links found on the page.</p></td></tr><tr>-<td><code>[options.userData]</code></td><td><code>Object</code></td><td></td>+<td><code>[options.transformRequestFunction]</code></td><td><code>function</code></td><td></td></tr><tr>-<td colspan="3"><p>An object that will be merged with the new <a href="request"><code>Request</code></a>'s <code>userData</code>, overriding any values that- were set via templating from <code>pseudoUrls</code>. This is useful when you need to override generic- <code>userData</code> set by the <a href="pseudourl"><code>PseudoUrl</code></a> template in specific use cases.</p>-<p> <strong>Example:</strong></p>-<pre><code>// pseudoUrl.userData-{- name: 'John',- surname: 'Doe',-}</code></pre><pre><code>// userData-{- name: 'Albert',- age: 31-}</code></pre><pre><code>// Enqueued request.userData-{- name: 'Albert',- surname: 'Doe',- age: 31,-}</code></pre></td></tr></tbody>+<td colspan="3"><p>Just before a new <a href="request"><code>Request</code></a> is constructed and enqueued to the <a href="requestqueue"><code>RequestQueue</code></a>, this function can be used+ to remove it or modify its contents such as <code>userData</code>, <code>payload</code> or, most importantly <code>uniqueKey</code>. This is useful+ when you need to enqueue multiple <code>Requests</code> to the queue that share the same URL, but differ in methods or payloads,+ or to dynamically update or create <code>userData</code>.</p>+<p> For example: by adding <code>keepUrlFragment: true</code> to the <code>request</code> object, URL fragments will not be removed+ when <code>uniqueKey</code> is computed.</p>+</td></tr></tbody></table><a name="utils.sleep"></a>diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.jsindex fb61d68a8..26982058c 100644--- a/src/enqueue_links/click_elements.js+++ b/src/enqueue_links/click_elements.js@@ -4,7 +4,7 @@ import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import log from 'apify-shared/log';import { RequestQueue, RequestQueueLocal } from '../request_queue';import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../puppeteer_request_interception';-import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches } from './shared';+import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches, createRequestOptions } from './shared';const STARTING_Z_INDEX = 2147400000;@@ -74,13 +74,14 @@ const STARTING_Z_INDEX = 2147400000;* If `pseudoUrls` is an empty array, `null` or `undefined`, then the function* enqueues all links found on the page.* @param {Function} [options.transformRequestFunction]- * Just before a new {@link Request} is enqueued to the {@link RequestQueue}, this function can be used to remove it- * or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful when you need- * to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,+ * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used+ * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful+ * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,* or to dynamically update or create `userData`.*- * For example: by generating your own `uniqueKey` from a combination of `url`, `method` and `payload` you enable crawling- * of websites that navigate using form submits (POST requests).+ * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from+ * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits+ * (POST requests).* @param {number} [options.waitForPageIdleSecs=1]* Clicking in the page triggers various asynchronous operations that lead to new URLs being shown* by the browser. It could be a simple JavaScript redirect or opening of a new tab in the browser.@@ -132,10 +133,11 @@ export async function enqueueLinksByClickingElements(options = {}) {waitForPageIdleMillis,maxWaitForPageIdleMillis,});- let requests = createRequests(interceptedRequests, pseudoUrlInstances);+ let requestOptions = createRequestOptions(interceptedRequests);if (transformRequestFunction) {- requests = requests.map(transformRequestFunction).filter(r => !!r);+ requestOptions = requestOptions.map(transformRequestFunction).filter(r => !!r);}+ const requests = createRequests(requestOptions, pseudoUrlInstances);return addRequestsToQueueInBatches(requests, requestQueue);}diff --git a/src/enqueue_links/enqueue_links.js b/src/enqueue_links/enqueue_links.jsindex c37770ba4..507bc1164 100644--- a/src/enqueue_links/enqueue_links.js+++ b/src/enqueue_links/enqueue_links.js@@ -3,7 +3,7 @@ import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import { RequestQueue, RequestQueueLocal } from '../request_queue';-import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches } from './shared';+import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatches, createRequestOptions } from './shared';/*** The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default)@@ -62,56 +62,33 @@ import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatche** If `pseudoUrls` is an empty array, `null` or `undefined`, then the function* enqueues all links found on the page.- * @param {Object} [options.userData]- * An object that will be merged with the new {@link Request}'s `userData`, overriding any values that- * were set via templating from `pseudoUrls`. This is useful when you need to override generic- * `userData` set by the {@link PseudoUrl} template in specific use cases.+ * @param {Function} [options.transformRequestFunction]+ * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used+ * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful+ * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,+ * or to dynamically update or create `userData`.*- * **Example:**- * ```- * // pseudoUrl.userData- * {- * name: 'John',- * surname: 'Doe',- * }- * ```- * ```- * // userData- * {- * name: 'Albert',- * age: 31- * }- * ```- * ```- * // Enqueued request.userData- * {- * name: 'Albert',- * surname: 'Doe',- * age: 31,- * }- * ```+ * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed+ * when `uniqueKey` is computed.* @return {Promise<QueueOperationInfo[]>}* Promise that resolves to an array of {@link QueueOperationInfo} objects.* @memberOf utils* @name enqueueLinks*/-export async function enqueueLinks(...args) {- // TODO: Remove after v1.0.0 gets released.- // Refactor enqueueLinks to use an options object and keep backwards compatibility- let page, $, selector, requestQueue, baseUrl, pseudoUrls, userData; // eslint-disable-line- if (args.length === 1) {- [{ page, $, selector = 'a', requestQueue, baseUrl, pseudoUrls, userData = {} }] = args;- } else {- [page, selector = 'a', requestQueue, pseudoUrls, userData = {}] = args;- log.deprecated('Passing individual arguments to enqueueLinks() is deprecated. '- + 'Use an options object: enqueueLinks({ page, selector, requestQueue, pseudoUrls, userData }) instead.');- }+export async function enqueueLinks(options = {}) {+ const {+ page,+ $,+ selector = 'a',+ requestQueue,+ baseUrl,+ pseudoUrls,+ userData, // TODO DEPRECATED 2019/06/27+ transformRequestFunction,+ } = options;- // Check for pseudoUrls as a third parameter.- if (Array.isArray(requestQueue)) {- const tmp = requestQueue;- requestQueue = pseudoUrls;- pseudoUrls = tmp;+ if (userData) {+ log.deprecated('options.userData of Apify.utils.enqueueLinks() is deprecated. Use options.transformRequestFunction instead.');}checkParamOrThrow(page, 'page', 'Maybe Object');@@ -127,13 +104,18 @@ export async function enqueueLinks(...args) {checkParamOrThrow(baseUrl, 'baseUrl', 'Maybe String');if (baseUrl && page) log.warning('The parameter options.baseUrl can only be used when parsing a Cheerio object. It will be ignored.');checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Maybe Array');- checkParamOrThrow(userData, 'userData', 'Object');+ checkParamOrThrow(userData, 'userData', 'Maybe Object');+ checkParamOrThrow(transformRequestFunction, 'transformRequestFunction', 'Maybe Function');// Construct pseudoUrls from input where necessary.const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls || []);const urls = page ? await extractUrlsFromPage(page, selector) : extractUrlsFromCheerio($, selector, baseUrl);- const requests = createRequests(urls, pseudoUrlInstances, userData);+ let requestOptions = createRequestOptions(urls, userData);+ if (transformRequestFunction) {+ requestOptions = requestOptions.map(transformRequestFunction).filter(r => !!r);+ }+ const requests = createRequests(requestOptions, pseudoUrlInstances);return addRequestsToQueueInBatches(requests, requestQueue);}diff --git a/src/enqueue_links/shared.js b/src/enqueue_links/shared.jsindex 5d39e5e24..d17d794f8 100644--- a/src/enqueue_links/shared.js+++ b/src/enqueue_links/shared.js@@ -45,43 +45,49 @@ export function constructPseudoUrlInstances(pseudoUrls) {});}/**- * @param {string[]|Object[]} sources+ * @param {string[]|Object[]} requestOptions* @param {PseudoUrl[]} pseudoUrls- * @param {Object} [userData]* @return {Request[]}* @ignore*/-export function createRequests(sources, pseudoUrls, userData = {}) {- const normalizedSources = sources- .map((src) => {- return typeof src === 'string'- ? { url: src, userData }- : { ...src, userData };- })- .filter(({ url }) => {- try {- return new URL(url).href;- } catch (err) {- return false;- }- });-+export function createRequests(requestOptions, pseudoUrls) {if (!(pseudoUrls && pseudoUrls.length)) {- return normalizedSources.map(src => new Request(src));+ return requestOptions.map(opts => new Request(opts));}const requests = [];- normalizedSources.forEach((src) => {+ requestOptions.forEach((opts) => {pseudoUrls- .filter(purl => purl.matches(src.url))+ .filter(purl => purl.matches(opts.url)).forEach((purl) => {- const request = purl.createRequest(src);+ const request = purl.createRequest(opts);requests.push(request);});});return requests;}+/**+ * @param {string[]|Object[]} sources+ * @param {Object} [userData]+ * @ignore+ */+export function createRequestOptions(sources, userData = {}) {+ return sources+ .map(src => (typeof src === 'string' ? { url: src } : src))+ .filter(({ url }) => {+ try {+ return new URL(url).href;+ } catch (err) {+ return false;+ }+ })+ .map((rqOpts) => {+ rqOpts.userData = { ...rqOpts.userData, ...userData };+ return rqOpts;+ });+}+/*** @param {Request[]} requests* @param {RequestQueue} requestQueuediff --git a/test/enqueue_links/enqueue_links.js b/test/enqueue_links/enqueue_links.jsindex 41e5cfeb7..3f129496d 100644--- a/test/enqueue_links/enqueue_links.js+++ b/test/enqueue_links/enqueue_links.js@@ -254,45 +254,6 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).to.be.eql({});});- // TODO Remove with 1.0.0- it('works with individual args instead of options object', async () => {- const enqueued = [];- const queue = new RequestQueue('xxx');- queue.addRequest = async (request) => {- enqueued.push(request);- };- const purls = [- new Apify.PseudoUrl('https://example.com/[(\\w|-|/)*]', { method: 'POST' }),- new Apify.PseudoUrl('[http|https]://cool.com/', { userData: { foo: 'bar' } }),- ];-- const originalLogDeprecated = log.deprecated;- const logOutput = [];- log.deprecated = (item) => { logOutput.push(item); };-- try {- await enqueueLinks(page, '.click', purls, queue, { hello: 'world' });-- expect(enqueued).to.have.lengthOf(3);-- expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');- expect(enqueued[0].method).to.be.eql('POST');- expect(enqueued[0].userData).to.be.eql({ hello: 'world' });-- expect(enqueued[1].url).to.be.eql('https://example.com/a/b/third');- expect(enqueued[1].method).to.be.eql('POST');- expect(enqueued[1].userData).to.be.eql({ hello: 'world' });-- expect(enqueued[2].url).to.be.eql('http://cool.com/');- expect(enqueued[2].method).to.be.eql('GET');- expect(enqueued[2].userData).to.be.eql({ hello: 'world', foo: 'bar' });- } finally {- log.deprecated = originalLogDeprecated;- }-- expect(logOutput.length).to.be.eql(1);- expect(logOutput[0]).to.include('options object');- });it('throws with sparse pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');diff --git a/test/enqueue_links/shared.js b/test/enqueue_links/shared.jsindex ca31947c1..d646863c3 100644--- a/test/enqueue_links/shared.js+++ b/test/enqueue_links/shared.js@@ -41,7 +41,8 @@ describe('Enqueue links shared functions', () => {];const userData = { bar: 'foo' };- const requests = shared.createRequests(sources, pseudoUrls, userData);+ const requestOptions = shared.createRequestOptions(sources, userData);+ const requests = shared.createRequests(requestOptions, pseudoUrls);expect(requests).to.have.lengthOf(2);requests.forEach((r) => {