Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Bump version, update Changelog, turn off reusePages
Ondra Urban762d1c141c11
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 24408fedf..519a91849 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,8 @@+0.11.5 / 2018-01-18+===================+- Disabled the `PuppeteerPool` `reusePages` option for now, due to a memory leak.+- Added a `keyValueStore.forEachKey()` method to iterate all keys in the store.+0.11.4 / 2018-01-15===================- Improvements in `Apify.utils.social.parseHandlesFromHtml` and `Apify.utils.htmlToText`diff --git a/package.json b/package.jsonindex 075da7afd..7f275309c 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.11.4",+ "version": "0.11.5","description": "The scalable web crawling and scraping library for JavaScript. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=8.14.0"diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex e03bb4b81..5db176c02 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -147,10 +147,6 @@ const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.* Always set this value in order to prevent infinite loops in misconfigured crawlers.* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.- * @param {boolean} [options.reusePages=false]- * Individual browser tabs will be reused after their task is complete instead- * of closing them and spawning a new tab. This saves CPU resources.- * See `reusePages` parameter of {@link PuppeteerPool}.* @param {Number} [options.maxOpenPagesPerInstance=50]* Maximum number of opened tabs per browser. If this limit is reached then a new* browser instance is started. See `maxOpenPagesPerInstance` parameter of {@link PuppeteerPool}.diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 744d0f57e..6abd61a92 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -103,10 +103,6 @@ class PuppeteerInstance {* ```* @param {Object} [options] All `PuppeteerPool` parameters are passed* via an options object with the following keys:- * @param {boolean} [options.reusePages=false]- * Individual browser tabs will be reused after their task is complete instead- * of closing them and spawning a new tab. This saves CPU resources.- * Try disabling this feature if you encounter any weird behavior in your crawlers.* @param {Number} [options.maxOpenPagesPerInstance=50]* Maximum number of open pages (i.e. tabs) per browser. When this limit is reached, new pages are loaded in a new browser instance.* @param {Number} [options.retireInstanceAfterRequestCount=100]@@ -145,7 +141,6 @@ class PuppeteerPool {}const {- reusePages,maxOpenPagesPerInstance,retireInstanceAfterRequestCount,launchPuppeteerFunction,@@ -159,6 +154,8 @@ class PuppeteerPool {// Disabling the feature since tests started failing.// Re-enable once the issues upstream are fixed.const recycleDiskCache = false;+ // Disabling due to memory leak.+ const reusePages = false;checkParamOrThrow(reusePages, 'options.reusePages', 'Boolean');checkParamOrThrow(maxOpenPagesPerInstance, 'options.maxOpenPagesPerInstance', 'Number');diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 7f31e40c7..8e40102b9 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -318,7 +318,7 @@ describe('PuppeteerPool', () => {});describe('reuse of browser tabs', () => {- it('should work', async () => {+ xit('should work', async () => {const pool = new Apify.PuppeteerPool({reusePages: true,});@@ -349,7 +349,7 @@ describe('PuppeteerPool', () => {await pool.destroy();});- it('should not open new browsers when idle pages are available', async () => {+ xit('should not open new browsers when idle pages are available', async () => {const pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,reusePages: true,@@ -381,7 +381,7 @@ describe('PuppeteerPool', () => {await pool.destroy();});- it('should count towards retireInstanceAfterRequestCount option', async () => {+ xit('should count towards retireInstanceAfterRequestCount option', async () => {const pool = new Apify.PuppeteerPool({retireInstanceAfterRequestCount: 2,reusePages: true,@@ -414,7 +414,7 @@ describe('PuppeteerPool', () => {await pool.destroy();});- it('should close pages in retired instances', async () => {+ xit('should close pages in retired instances', async () => {const pool = new Apify.PuppeteerPool({retireInstanceAfterRequestCount: 1,reusePages: true,@@ -448,7 +448,7 @@ describe('PuppeteerPool', () => {await pool.destroy();});- it('should skip closed pages', async () => {+ xit('should skip closed pages', async () => {const pool = new Apify.PuppeteerPool({ reusePages: true });const firstPage = await pool.newPage();