Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of PuppeteerPool recycleDiskCache option
Jan Curn8e3c9c63cb91
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 7d03a61aa..719393790 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -426,7 +426,7 @@ export default class PuppeteerPool {.then(() => {// Delete all cache directoriesconst promises = [];- while (this.recycledDiskCacheDirs.length > 0) {+ while (this.recycledDiskCacheDirs && this.recycledDiskCacheDirs.length > 0) {promises.push(deleteDiskCacheDir(this.recycledDiskCacheDirs.removeFirst()));}return Promise.all(promises);diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex bea607f6b..bc028ce18 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -294,7 +294,6 @@ describe('PuppeteerPool', () => {expect(cookies2after.length).to.be.at.least(1);// TODO: This only works in headful mode now, but must work all the time- // Seeif (isMacOs) {expect(fromDiskCache2).to.be.at.least(1);}