Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix merge conflicts
Ondra Urban123308bb4b21
diff --cc package.jsonindex 12c676239,e3d8f4e9b..2665f2c5f--- a/package.json+++ b/package.json@@@ -1,11 -1,11 +1,11 @@@{"name": "apify",- "version": "0.5.51",+ "version": "0.6.1","description": "Web scraping and automation SDK",- "main": "build/index.js","engines": {- "node": ">=7.6.0"+ "node": ">=8.0.0"},+ "main": "build/index.js","keywords": ["apify","headless",diff --cc src/puppeteer_crawler.jsindex 9241bfda5,16b9197ad..cfb77dde7--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@@ -231,10 -214,9 +231,10 @@@ export default class PuppeteerCrawler* @ignore*/_handleRequestFunction({ request }) {+ if (this.isStopped) return Promise.reject(new Error('PuppeteerCrawler is stopped.')); // Pool will be destroyed.let page;- const handlePagePromise = this.puppeteerPool- return this.puppeteerPool++ const promise = this.puppeteerPool.newPage().then((newPage) => { page = newPage; }).then(() => this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool }))@@@ -260,6 -241,5 +259,6 @@@.timeout(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.').catch(err => log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err ? err.message : err }));});- return Promise.race([finalPromise, this.isRunningPromise]); // Reject all requests immediately after stopping the crawler (to be reclaimed).++ return Promise.race([promise, this.isRunningPromise]); // Reject all requests immediately after stopping the crawler (to be reclaimed).}}