Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix merge conflicts

Ondra Urban123308bb4b21
diff --cc package.json
index 12c676239,e3d8f4e9b..2665f2c5f
--- a/package.json
+++ b/package.json
@@@ -1,11 -1,11 +1,11 @@@
{
"name": "apify",
- "version": "0.5.51",
+ "version": "0.6.1",
"description": "Web scraping and automation SDK",
- "main": "build/index.js",
"engines": {
- "node": ">=7.6.0"
+ "node": ">=8.0.0"
},
+ "main": "build/index.js",
"keywords": [
"apify",
"headless",
diff --cc src/puppeteer_crawler.js
index 9241bfda5,16b9197ad..cfb77dde7
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@@ -231,10 -214,9 +231,10 @@@ export default class PuppeteerCrawler
* @ignore
*/
_handleRequestFunction({ request }) {
+ if (this.isStopped) return Promise.reject(new Error('PuppeteerCrawler is stopped.')); // Pool will be destroyed.
let page;
- const handlePagePromise = this.puppeteerPool
- return this.puppeteerPool
++ const promise = this.puppeteerPool
.newPage()
.then((newPage) => { page = newPage; })
.then(() => this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool }))
@@@ -260,6 -241,5 +259,6 @@@
.timeout(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')
.catch(err => log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err ? err.message : err }));
});
- return Promise.race([finalPromise, this.isRunningPromise]); // Reject all requests immediately after stopping the crawler (to be reclaimed).
++ return Promise.race([promise, this.isRunningPromise]); // Reject all requests immediately after stopping the crawler (to be reclaimed).
}
}