Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix tests
Ondra Urban559027cf7174
diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 34fcd6a59..b70cbb0e9 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -206,7 +206,7 @@ export const launchPuppeteer = (options = {}) => {checkParamOrThrow(options.proxyUrl, 'options.proxyUrl', 'Maybe String');checkParamOrThrow(options.useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');checkParamOrThrow(options.liveView, 'options.liveView', 'Maybe Boolean');- checkParamOrThrow(options.liveViewOptions, 'options.liveViewOptoins', 'Maybe Object');+ checkParamOrThrow(options.liveViewOptions, 'options.liveViewOptions', 'Maybe Object');if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');const puppeteer = getPuppeteerOrThrow();diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex ec1d93086..201be21ce 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -13,6 +13,7 @@ import { addTimeoutToPromise } from './utils';/* global process */const PROCESS_KILL_TIMEOUT_MILLIS = 5000;+const OPEN_NEW_TAB_RETRIES = 2;const DEFAULT_OPTIONS = {reusePages: false,@@ -460,7 +461,7 @@ class PuppeteerPool {}}// If there are no live pages to be reused, we spawn a new tab.- return this._openNewTab(2);+ return this._openNewTab(OPEN_NEW_TAB_RETRIES);}/**@@ -497,9 +498,7 @@ class PuppeteerPool {return new Promise((resolve) => {// We can do this because _retireInstance is synchronous// so the page will open in a different browser.- setImmediate(() => {- resolve(this._openNewTab(--retryCount));- });+ resolve(this._openNewTab(--retryCount));});}const betterError = new Error(`PuppeteerPool: browser.newPage() failed: ${instance.id}.`);diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 2e9597bf0..ee377ef99 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -155,6 +155,9 @@ describe('CheerioCrawler', () => {handleFailedRequestFunction: ({ request }) => failed.push(request),});+ // Override low value to prevent seeing timeouts from BasicCrawler+ cheerioCrawler.basicCrawler.handleRequestTimeoutMillis = 10000;+await requestList.initialize();await cheerioCrawler.run();