Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixes of recycleDiskCache option in PuppeteerPool
Jan Curn2e486d7d576f
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 996323fc4..3f48911b4 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -40,7 +40,7 @@ const DISK_CACHE_DIR = path.join(os.tmpdir(), 'puppeteer_disk_cache-');*/const deleteDiskCacheDir = (diskCacheDir) => {log.debug('PuppeteerPool: Deleting disk cache directory', { diskCacheDir });- rimrafAsync(diskCacheDir)+ return rimrafAsync(diskCacheDir).catch((err) => {log.warning('PuppeteerPool: Cannot delete Chrome disk cache directory', { diskCacheDir, errorMessage: err.message });});@@ -151,6 +151,15 @@ export default class PuppeteerPool {checkParamOrThrow(launchPuppeteerOptions, 'opts.launchPuppeteerOptions', 'Maybe Object');checkParamOrThrow(recycleDiskCache, 'opts.recycleDiskCache', 'Maybe Boolean');+ // The recycleDiskCache option is only supported in headful mode+ // See https://bugs.chromium.org/p/chromium/issues/detail?id=882431+ if (recycleDiskCache+ && ((!launchPuppeteerOptions+ || (launchPuppeteerOptions && launchPuppeteerOptions.headless)+ || (launchPuppeteerOptions && launchPuppeteerOptions.headless === undefined && !launchPuppeteerOptions.devtools)))) {+ log.warning('PuppeteerPool: The "recycleDiskCache" is currently only supported in headful mode. Disk cache will not be recycled.');+ }+// Config.this.maxOpenPagesPerInstance = maxOpenPagesPerInstance;this.retireInstanceAfterRequestCount = retireInstanceAfterRequestCount;@@ -419,7 +428,7 @@ export default class PuppeteerPool {return browser;}).then((browser) => {- if (browser.recycleDiskCacheDir) deleteDiskCacheDir(browser.recycleDiskCacheDir);+ if (browser.recycleDiskCacheDir) return deleteDiskCacheDir(browser.recycleDiskCacheDir);});});diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex bc9829d76..e32546c2e 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -231,7 +231,9 @@ describe('PuppeteerPool', () => {});it('supports recycleDiskCache option', async () => {- const isMacOs = false; // process.platform === 'darwin';+ // NOTE: This feature only works in headful mode now+ // See https://bugs.chromium.org/p/chromium/issues/detail?id=882431+ const isMacOs = process.platform === 'darwin';const pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,@@ -293,8 +295,6 @@ describe('PuppeteerPool', () => {const cookies2after = await page2.cookies(url);expect(cookies2after.length).to.be.at.least(1);- // TODO: This only works in headful mode now, but must work all the time- // See https://bugs.chromium.org/p/chromium/issues/detail?id=882431if (isMacOs) {expect(fromDiskCache2).to.be.at.least(1);}@@ -311,7 +311,6 @@ describe('PuppeteerPool', () => {await pool.destroy();// Check cache dirs were deleted- await Apify.utils.sleep(1000);expect(fs.existsSync(dir1)).to.be.eql(false);expect(fs.existsSync(dir3)).to.be.eql(false);});