Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed navigation timeout session retirement (#892)
petrpatekc197fbea8e30
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.jsindex 1b9ff3b50..8e780c063 100644--- a/src/crawlers/browser_crawler.js+++ b/src/crawlers/browser_crawler.js@@ -9,6 +9,7 @@ import BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-dupimport { validators } from '../validators';import {throwOnBlockedRequest,+ handleRequestTimeout,} from './crawler_utils';// eslint-enable-line import/no-duplicates@@ -315,7 +316,7 @@ class BrowserCrawler extends BasicCrawler {log: this.log,});- // Assuming there are not more thant 20 browsers running at once;+ // Assuming there are not more than 20 browsers running at once;this.sessionPool.setMaxListeners(20);}@@ -374,12 +375,7 @@ class BrowserCrawler extends BasicCrawler {if (session) session.markGood();} finally {- try {- await page.close();- } catch (error) {- // Only log error in page close.- this.log.debug('Error while closing page', { error });- }+ page.close().catch((error) => this.log.debug('Error while closing page', { error }));}}@@ -398,8 +394,6 @@ class BrowserCrawler extends BasicCrawler {crawlingContext.session = browserControllerInstance.launchContext.session;crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;-- crawlingContext.crawler = this;}/**@@ -411,11 +405,30 @@ class BrowserCrawler extends BasicCrawler {// @TODO: consider deep cloneconst gotoOptions = _.clone(this.defaultGotoOptions);await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);- crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);+ try {+ crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);+ } catch (error) {+ this._handleNavigationTimeout(crawlingContext, error);++ throw error;+ }await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);}+ /**+ * Marks session bad in case of navigation timeout.+ * @param {object} crawlingContext+ * @param {Error} error+ */+ _handleNavigationTimeout(crawlingContext, error) {+ const { session } = crawlingContext;++ if (error && error.constructor.name === 'TimeoutError') {+ handleRequestTimeout(session, error.message);+ }+ }+/**** @param {object} crawlingContextdiff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.jsindex 5b1551fa7..331e5cb4e 100644--- a/src/crawlers/playwright_crawler.js+++ b/src/crawlers/playwright_crawler.js@@ -1,7 +1,6 @@import { PlaywrightPlugin } from 'browser-pool';import ow from 'ow';import BrowserCrawler from './browser_crawler';-import { handleRequestTimeout } from './crawler_utils';import { gotoExtended } from '../playwright_utils';import { apifyOptionsToLaunchOptions, getPlaywrightLauncherOrThrow } from '../playwright';@@ -258,17 +257,12 @@ class PlaywrightCrawler extends BrowserCrawler {browserPoolOptions,});+ this.gotoTimeoutMillis = gotoTimeoutSecs * 1000;+if (gotoTimeoutSecs) {this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');}- this.browserPool.postLaunchHooks.push(({ error, session }) => {- // It would be better to compare the instances,- if (error && error.constructor.name === 'TimeoutError') {- handleRequestTimeout(session, error.message);- }- });-this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;this.launchContext = launchContext;this.playwrightModule = playwrightModule;diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 1ac5f8b7e..d37e21fbe 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -2,7 +2,6 @@ import { PuppeteerPlugin } from 'browser-pool';import ow from 'ow';import BrowserCrawler from './browser_crawler';-import { handleRequestTimeout } from './crawler_utils';import { gotoExtended } from '../puppeteer_utils';import { apifyOptionsToLaunchOptions, getPuppeteerOrThrow } from '../puppeteer';import applyStealthToBrowser from '../stealth/stealth';@@ -273,14 +272,6 @@ class PuppeteerCrawler extends BrowserCrawler {browserPoolOptions,});- this.browserPool.postLaunchHooks.push(({ error, session }) => {- // It would be better to compare the instances,- // but we don't have access to puppeteer.errors here.- if (error && error.constructor.name === 'TimeoutError') {- handleRequestTimeout(session, error.message);- }- });-if (gotoTimeoutSecs) {this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');}diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.jsindex 94c7c8a45..80257b77f 100644--- a/test/crawlers/browser_crawler.test.js+++ b/test/crawlers/browser_crawler.test.js@@ -112,6 +112,38 @@ describe('BrowserCrawler', () => {expect(browserCrawler.browserPool.destroy).toBeCalled();});+ test('should retire session after TimeouError', async () => {+ const requestList = new Apify.RequestList({+ sources: [+ { url: 'http://example.com/?q=1' },+ ],+ });+ class TimeoutError extends Error {++ }+ let sessionGoto;+ const browserCrawler = new Apify.BrowserCrawler({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ },+ requestList,+ useSessionPool: true,+ handlePageFunction: async () => {+ return Promise.resolve();+ },+ maxRequestRetries: 1,+ gotoFunction: async ({ session }) => {+ jest.spyOn(session, 'markBad');+ sessionGoto = session;+ throw new TimeoutError();+ },+ });++ await requestList.initialize();+ await browserCrawler.run();+ expect(sessionGoto.markBad).toBeCalled();+ });+test('should evaluate preNavigationHooks', async () => {const requestList = new Apify.RequestList({sources: [