Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added Apify.utils.puppeteer.blockResources() - Eureka!
Jan Curn1ca3a8684abf
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 376eefb9a..f45626fa9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,7 @@xxx===================- PuppeteerCrawler's default page load timeout changed from 30 to 60 seconds.+- Added `Apify.utils.puppeteer.blockResources()` function0.5.51 / 2018-08-09===================diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 75864adbc..628cd48bf 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -162,6 +162,35 @@ const enqueueLinks = async (page, selector, purls, requestQueue) => {return Promise.mapSeries(requests, request => requestQueue.addRequest(request));};+/**+ * Forces the browser tab to block loading certain page resources,+ * using the `Page.setRequestInterception(value)` method.+ * This is useful to speed up crawling of websites.+ *+ * The resource types to block can be controlled using the `resourceTypes` parameter,+ * which indicates the types of resources as they are perceived by the rendering engine.+ * The following resource types are currently supported:+ * `document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`,+ * `eventsource`, `websocket`, `manifest`, `other`.+ * For more details, see Puppeteer's+ * <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#requestresourcetype">Request.resourceType() documentation</a>.+ *+ * By default, the function blocks these resource types: `stylesheet`, `font`, `image`, `media`.+ *+ * @param {Page} page Puppeteer's `Page` object+ * @param {Array<String>} resourceTypes Array of resource types to block.+ * @return {Promise<void>}+ */+const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {+ await page.setRequestInterception(true);+ page.on('request', async (request) => {+ const type = request.resourceType();+ if (resourceTypes.includes(type)) request.abort();+ else request.continue();+ });+};++/*** A namespace that contains various Puppeteer utilities.*@@ -187,4 +216,5 @@ export const puppeteerUtils = {injectUnderscore,enqueueRequestsFromClickableElements,enqueueLinks,+ blockResources,};diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex 56909f5ed..56c8b063d 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -195,4 +195,32 @@ describe('Apify.utils.puppeteer', () => {browser.close();}});++ it('supports blockResources() with default values', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ try {+ const page = await browser.newPage();+ await Apify.utils.puppeteer.blockResources(page);+ await page.goto('about:blank');++ // TODO: Write some proper unit test for this+ } finally {+ browser.close();+ }+ });++ it('supports blockResources() with nondefault values', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ try {+ const page = await browser.newPage();+ await Apify.utils.puppeteer.blockResources(page, ['font']);+ await page.goto('about:blank');++ // TODO: Write some proper unit test for this+ } finally {+ browser.close();+ }+ });});