Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
added timeout session retire.
Petr Patek0a6d102cb492
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex bcdd6142a..1aaf314cd 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -440,7 +440,7 @@ class CheerioCrawler {responseStream = await requestAsBrowser(opts);} catch (e) {if (e instanceof TimeoutError) {- throw new Error(`CheerioCrawler: request timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);+ this._handleRequestTimeout(session);} else {throw e;}@@ -593,6 +593,16 @@ class CheerioCrawler {if (session) session.retire();throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);}++ /**+ * Handles timeout request+ * @param session {Session}+ * @private+ */+ _handleRequestTimeout(session) {+ if (session) session.markBad();+ throw new Error(`CheerioCrawler: request timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);+ }}diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 79cf8decf..94840d5bb 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -820,6 +820,32 @@ describe('CheerioCrawler', () => {await crawler.run();});+ test('should markBad sessions after request timeout', async () => {+ const sessions = [];+ const failed = [];+ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList: await Apify.openRequestList('timeoutTest', [`http://${HOST}:${port}/timeout?a=12`,+ `http://${HOST}:${port}/timeout?a=23`,+ ]),+ maxRequestRetries: 1,+ requestTimeoutSecs: 1,+ maxConcurrency: 1,+ useSessionPool: true,+ handlePageFunction: async () => {},+ handleFailedRequestFunction: ({ request }) => failed.push(request),+ });+ const oldCall = cheerioCrawler._handleRequestTimeout;+ cheerioCrawler._handleRequestTimeout = (session) => {+ sessions.push(session);+ return oldCall(session).bind(cheerioCrawler);+ };++ await cheerioCrawler.run();+ sessions.forEach((session) => {+ expect(session.errorScore).toEqual(1);+ });+ });+test('should retire session on "blocked" status codes', async () => {for (const code of STATUS_CODES_BLOCKED) {const failed = [];