Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixes of recycleDiskCache option in PuppeteerPool

Jan Curn2e486d7d576f
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 996323fc4..3f48911b4 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -40,7 +40,7 @@ const DISK_CACHE_DIR = path.join(os.tmpdir(), 'puppeteer_disk_cache-');
*/
const deleteDiskCacheDir = (diskCacheDir) => {
log.debug('PuppeteerPool: Deleting disk cache directory', { diskCacheDir });
- rimrafAsync(diskCacheDir)
+ return rimrafAsync(diskCacheDir)
.catch((err) => {
log.warning('PuppeteerPool: Cannot delete Chrome disk cache directory', { diskCacheDir, errorMessage: err.message });
});
@@ -151,6 +151,15 @@ export default class PuppeteerPool {
checkParamOrThrow(launchPuppeteerOptions, 'opts.launchPuppeteerOptions', 'Maybe Object');
checkParamOrThrow(recycleDiskCache, 'opts.recycleDiskCache', 'Maybe Boolean');
+ // The recycleDiskCache option is only supported in headful mode
+ // See https://bugs.chromium.org/p/chromium/issues/detail?id=882431
+ if (recycleDiskCache
+ && ((!launchPuppeteerOptions
+ || (launchPuppeteerOptions && launchPuppeteerOptions.headless)
+ || (launchPuppeteerOptions && launchPuppeteerOptions.headless === undefined && !launchPuppeteerOptions.devtools)))) {
+ log.warning('PuppeteerPool: The "recycleDiskCache" is currently only supported in headful mode. Disk cache will not be recycled.');
+ }
+
// Config.
this.maxOpenPagesPerInstance = maxOpenPagesPerInstance;
this.retireInstanceAfterRequestCount = retireInstanceAfterRequestCount;
@@ -419,7 +428,7 @@ export default class PuppeteerPool {
return browser;
})
.then((browser) => {
- if (browser.recycleDiskCacheDir) deleteDiskCacheDir(browser.recycleDiskCacheDir);
+ if (browser.recycleDiskCacheDir) return deleteDiskCacheDir(browser.recycleDiskCacheDir);
});
});
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index bc9829d76..e32546c2e 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -231,7 +231,9 @@ describe('PuppeteerPool', () => {
});
it('supports recycleDiskCache option', async () => {
- const isMacOs = false; // process.platform === 'darwin';
+ // NOTE: This feature only works in headful mode now
+ // See https://bugs.chromium.org/p/chromium/issues/detail?id=882431
+ const isMacOs = process.platform === 'darwin';
const pool = new Apify.PuppeteerPool({
maxOpenPagesPerInstance: 1,
@@ -293,8 +295,6 @@ describe('PuppeteerPool', () => {
const cookies2after = await page2.cookies(url);
expect(cookies2after.length).to.be.at.least(1);
- // TODO: This only works in headful mode now, but must work all the time
- // See https://bugs.chromium.org/p/chromium/issues/detail?id=882431
if (isMacOs) {
expect(fromDiskCache2).to.be.at.least(1);
}
@@ -311,7 +311,6 @@ describe('PuppeteerPool', () => {
await pool.destroy();
// Check cache dirs were deleted
- await Apify.utils.sleep(1000);
expect(fs.existsSync(dir1)).to.be.eql(false);
expect(fs.existsSync(dir3)).to.be.eql(false);
});