Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Remove interception from code base

Ondra Urban4db6a0870e04
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 47560efa7..700ce1af4 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -5,10 +5,9 @@ import BasicCrawler from './basic_crawler';
import PuppeteerPool from './puppeteer_pool';
import { addTimeoutToPromise } from './utils';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';
-import { gotoExtended } from './puppeteer_utils';
const DEFAULT_OPTIONS = {
- gotoFunction: async ({ request, page }) => gotoExtended(page, request, { timeout: 60000 }),
+ gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),
handlePageTimeoutSecs: 60,
handleFailedRequestFunction: ({ request }) => {
const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 3485e4c88..36a06dc55 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -188,24 +188,27 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque
/**
- * Forces the Puppeteer browser tab to block loading certain HTTP resources.
+ * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
* This is useful to speed up crawling of websites, since it reduces the amount
- * of data that need to be downloaded from the web.
+ * of data that needs to be downloaded from the web, but it may break some websites
+ * or unexpectedly prevent loading of resources.
*
- * The resource types to block can be specified using the `resourceTypes` parameter,
- * which indicates the types of resources as they are perceived by the rendering engine.
- * The following resource types are currently supported:
- * `document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`,
- * `eventsource`, `websocket`, `manifest`, `other`.
- * For more details, see Puppeteer's
- * <a href="https://pptr.dev/#?product=Puppeteer&show=api-requestresourcetype" target="_blank">Request.resourceType() documentation</a>.
+ * If the `options.urlPatterns` parameter is not provided,
+ * by default the function blocks URLs that include these patterns:
*
- * If the `resourceTypes` parameter is not provided,
- * by default the function blocks these resource types: `stylesheet`, `font`, `image`, `media`.
+ * ```json
+ * [".css", ".jpg", ".jpeg", ".png", ".svg", ".woff", ".pdf", ".zip"]
+ * ```
+ *
+ * The defaults will be concatenated with the patterns you provide in `options.urlPatterns`.
+ * If you want to remove the defaults, use `options.includeDefaults: false`.
+ *
+ * This function does not use Puppeteer's request interception and therefore does not interfere
+ * with browser cache. It's also faster than blocking requests using interception,
+ * because the blocking happens directly in the browser without the round-trip to Node.js,
+ * but it does not provide the extra benefits of request interception.
*
- * Note that the `blockResources` function internally uses Puppeteer's
- * [`Page.setRequestInterception()`](https://pptr.dev/#?product=Puppeteer&show=api-pagesetrequestinterceptionvalue) function,
- * which can only be used once per `Page` object.
+ * The function will never block main document loads and their respective redirects.
*
* **Example usage**
* ```javascript
@@ -214,23 +217,27 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque
* const browser = await Apify.launchPuppeteer();
* const page = await browser.newPage();
*
- * // Block all resources except for the main HTML document
- * await Apify.utils.puppeteer.blockResources(page,
- * ['stylesheet', 'image', 'media', 'font', 'script', 'texttrack', 'xhr',
- * 'fetch', 'eventsource', 'websocket', 'manifest', 'other']
- * );
+ * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
+ * await Apify.utils.puppeteer.blockRequests(page, {
+ * urlPatterns: ['adsbygoogle.js'],
+ * });
*
- * await page.goto('https://www.example.com');
+ * await page.goto('https://cnn.com');
* ```
*
* @param {Page} page
* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
- * @param {String[]} [resourceTypes=['stylesheet', 'font', 'image', 'media']]
- * Array of resource types to block.
+ * @param {Object} [options]
+ * @param {string[]} [options.urlPatterns]
+ * The patterns of URLs to block from being loaded by the browser.
+ * Only `*` can be used as a wildcard. It is also automatically added to the beginning
+ * and end of the pattern. This limitation is enforced by the DevTools protocol.
+ * `.png` is the same as `*.png*`.
+ * @param {boolean} [options.includeDefaults]
* @return {Promise}
* @memberOf puppeteer
*/
-const blockUrlPatterns = async (page, options = {}) => {
+const blockRequests = async (page, options = {}) => {
const {
urlPatterns = [],
includeDefaults = true,
@@ -238,12 +245,19 @@ const blockUrlPatterns = async (page, options = {}) => {
checkParamOrThrow(urlPatterns, 'options.urlPatterns', '[String]');
checkParamOrThrow(includeDefaults, 'options.includeDefaults', 'Boolean');
- checkParamOrThrow(blockOnRedirects, 'options.blockOnRedirects', 'Boolean');
+ const patternsToBlock = includeDefaults
+ ? [...DEFAULT_BLOCK_REQUEST_URL_PATTERNS, ...urlPatterns]
+ : urlPatterns;
+ await page._client.send('Network.setBlockedURLs', { urls: patternsToBlock }); // eslint-disable-line no-underscore-dangle
};
-
+/**
+ * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
+ * 'Until this resolves, please use `Apify.utils.puppeteer.blockRequests()`.
+ * @deprecated
+ */
const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
log.deprecated('Apify.utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. '
+ 'Until this resolves, please use Apify.utils.puppeteer.blockRequests()');
@@ -255,6 +269,9 @@ const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'imag
};
/**
+ * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
+ * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
+ *
* Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
* *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
* This issue should be resolved or atleast mitigated in future iterations of this feature.
@@ -267,12 +284,16 @@ const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'imag
* String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
* @return {Promise}
* @memberOf puppeteer
+ * @deprecated
*/
const cacheResponses = async (page, cache, responseUrlRules) => {
checkParamOrThrow(page, 'page', 'Object');
checkParamOrThrow(cache, 'cache', 'Object');
checkParamOrThrow(responseUrlRules, 'responseUrlRules', 'Array');
+ log.deprecated('Apify.utils.puppeteer.cacheResponses() has a high impact on performance '
+ + 'in recent versions of Puppeteer so it\'s use is discouraged until this issue resolves.');
+
// Check that rules are either String or RegExp
responseUrlRules.forEach((rule, index) => checkParamOrThrow(rule, `responseUrlRules[${index}]`, 'String | RegExp'));
@@ -361,6 +382,10 @@ const compileScript = (scriptString, context = Object.create(null)) => {
};
/**
+ * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
+ * performance. Until this resolves, we suggest using this function only when needed and not to replace all `page.goto()`
+ * calls in your actors with it.
+ *
* Extended version of Puppeteer's `page.goto()` allowing to perform requests with HTTP method other than GET,
* with custom headers and POST payload. URL, method, headers and payload are taken from
* request parameter that must be an instance of Apify.Request class.
@@ -379,6 +404,9 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {
checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');
checkParamOrThrow(gotoOptions, 'gotoOptions', 'Object');
+ log.deprecated('Apify.utils.puppeteer.gotoExtended() has a high impact on performance '
+ + 'in recent versions of Puppeteer. Use only when necessary.');
+
const { method, headers, payload } = request;
if (method !== 'GET' || payload || !_.isEmpty(headers)) {
@@ -405,20 +433,6 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {
return page.goto(request.url, gotoOptions);
};
-/*
-export const enqueueClickables = async (page, purls, selector) => {
- const interceptRequestHandler = async (interceptedRequest) => {
- // TODO: configure everything here
- };
-
- await addInterceptRequestHandler(page, interceptRequestHandler);
-
- // TODO: Click elements here
-
- await removeInterceptRequestHandler(page, interceptRequestHandler);
-};
-*/
-
let logEnqueueLinksDeprecationWarning = true;
/**
@@ -455,6 +469,7 @@ export const puppeteerUtils = {
return enqueueLinks(...args);
}
},
+ blockRequests,
blockResources,
cacheResponses,
compileScript,
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 455172de1..69c57360d 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -149,6 +149,52 @@ describe('Apify.utils.puppeteer', () => {
}
});
+ it('blockRequests() works with default values', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true });
+ const loadedUrls = [];
+
+ try {
+ const page = await browser.newPage();
+ await Apify.utils.puppeteer.blockRequests(page);
+ page.on('response', response => loadedUrls.push(response.url()));
+ await page.setContent(`<html><body>
+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">
+ <img src="https://example.com/image.png" />
+ <script src="https://example.com/script.js" defer="defer">></script>
+ </body></html>`, { waitUntil: 'networkidle0' });
+ } finally {
+ await browser.close();
+ }
+ expect(loadedUrls).to.have.members([
+ 'https://example.com/script.js',
+ ]);
+ });
+
+ it('blockRequests() works with overridden values', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true });
+ const loadedUrls = [];
+
+ try {
+ const page = await browser.newPage();
+ await Apify.utils.puppeteer.blockRequests(page, {
+ urlPatterns: ['.css'],
+ includeDefaults: false,
+ });
+ page.on('response', response => loadedUrls.push(response.url()));
+ await page.setContent(`<html><body>
+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">
+ <img src="https://example.com/image.png" />
+ <script src="https://example.com/script.js" defer="defer">></script>
+ </body></html>`, { waitUntil: 'networkidle0' });
+ } finally {
+ await browser.close();
+ }
+ expect(loadedUrls).to.have.members([
+ 'https://example.com/image.png',
+ 'https://example.com/script.js',
+ ]);
+ });
+
it('supports blockResources() with default values', async () => {
const browser = await Apify.launchPuppeteer({ headless: true });
const loadedUrls = [];