Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix session management in PuppeteerCrawler (#791)
Ondra Urban4c1ebef117ac
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6803e6c11..3d6a84011 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,4 +1,13 @@-0.21.5 / 2020/09/02+0.21.6 / 2020/09/01+====================+It appears that `CheerioCrawler` was correctly retiring sessions on timeouts+and blocked status codes (401, 403, 429), whereas `PuppeteerCrawler` did not.+Apologies for the omission, this release fixes the problem.++- Fix sessions not being retired on blocked status codes in `PuppeteerCrawler`.+- Fix sessions not being marked bad on navigation timeouts in `PuppeteerCrawler`.++0.21.5 / 2020/09/30====================This is a very minor release that fixes some issues that were preventinguse of the SDK with Node 14.diff --git a/package.json b/package.jsonindex 8b214481c..24fb031b1 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.21.5",+ "version": "0.21.6","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": "^10.17.0 || ^12.3.0"diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex f3a4d366f..946b4c205 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -370,9 +370,20 @@ class PuppeteerCrawler {proxyInfo = browserInstance.proxyInfo;}-try {- const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session, proxyInfo });+ let response;+ try {+ response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session, proxyInfo });+ } catch (err) {+ // It would be better to compare the instances,+ // but we don't have access to puppeteer.errors here.+ if (err.constructor.name === 'TimeoutError') {+ this._handleRequestTimeout(session, err.message);+ }+ }++ if (this.useSessionPool) this._throwOnBlockedRequest(session, response.status());+await this.puppeteerPool.serveLiveViewSnapshot(page);request.loadedUrl = page.url();@@ -416,6 +427,33 @@ class PuppeteerCrawler {const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');this.log.exception(error, 'Request failed and reached maximum retries', details);}++ /**+ * Handles timeout request+ * @param {Session} session+ * @param {string} errorMessage+ * @private+ */+ _handleRequestTimeout(session, errorMessage) {+ if (session) session.markBad();+ const timeoutMillis = errorMessage.match(/(\d+) ms/)[1]; // first capturing group+ const timeoutSecs = Number(timeoutMillis) / 1000;+ throw new Error(`gotoFunction timed out after ${timeoutSecs} seconds.`);+ }++ /**+ * Handles blocked request+ * @param {Session} session+ * @param {number} statusCode+ * @private+ */+ _throwOnBlockedRequest(session, statusCode) {+ const isBlocked = session.retireOnBlockedStatusCodes(statusCode);++ if (isBlocked) {+ throw new Error(`Request blocked - received ${statusCode} status code.`);+ }+ }}export default PuppeteerCrawler;diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex a786e698d..0f3966199 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -757,13 +757,14 @@ describe('CheerioCrawler', () => {handlePageFunction: async () => {},handleFailedRequestFunction: ({ request }) => failed.push(request),});- const oldCall = cheerioCrawler._handleRequestTimeout;+ const oldCall = cheerioCrawler._handleRequestTimeout.bind(cheerioCrawler);cheerioCrawler._handleRequestTimeout = (session) => {sessions.push(session);- return oldCall(session).bind(cheerioCrawler);+ return oldCall(session);};await cheerioCrawler.run();+ expect(sessions.length).toBe(4);sessions.forEach((session) => {expect(session.errorScore).toEqual(1);});@@ -790,7 +791,7 @@ describe('CheerioCrawler', () => {failed.push(request);},});- const oldCall = crawler._throwOnBlockedRequest;+ const oldCall = crawler._throwOnBlockedRequest.bind(crawler);crawler._throwOnBlockedRequest = (session, statusCode) => {sessions.push(session);return oldCall(session, statusCode);diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 402653409..51aca2219 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -2,6 +2,7 @@ import { ENV_VARS } from 'apify-shared/consts';import sinon from 'sinon';import log from '../../build/utils_log';import * as Apify from '../../build';+import { STATUS_CODES_BLOCKED } from '../../build/constants';import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utilsRequest from '../../build/utils_request';@@ -116,9 +117,9 @@ describe('PuppeteerCrawler', () => {handlePageSessions.push(session);return Promise.resolve();},- gotoFunction: async ({ session }) => {+ gotoFunction: async ({ session, page, request }) => {goToPageSessions.push(session);- return Apify.launchPuppeteer();+ return page.goto(request.url);},});@@ -164,6 +165,113 @@ describe('PuppeteerCrawler', () => {loadedCookies.forEach(cookie => expect(cookie).toEqual('TEST=12321312312'));});+ test('should throw on "blocked" status codes', async () => {+ const baseUrl = 'https://example.com/';+ const sources = STATUS_CODES_BLOCKED.map((statusCode) => {+ return {+ url: baseUrl + statusCode,+ userData: { statusCode },+ };+ });+ const requestList = await Apify.openRequestList(null, sources);++ let called = false;+ const failedRequests = [];+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ useSessionPool: true,+ persistCookiesPerSession: false,+ maxRequestRetries: 0,+ gotoFunction: async ({ request }) => {+ return { status: () => request.userData.statusCode };+ },+ handlePageFunction: async () => { // eslint-disable-line no-loop-func+ called = true;+ },+ handleFailedRequestFunction: async ({ request }) => {+ failedRequests.push(request);+ },+ });++ await crawler.run();++ expect(failedRequests.length).toBe(STATUS_CODES_BLOCKED.length);+ failedRequests.forEach((fr) => {+ const [msg] = fr.errorMessages;+ expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);+ });+ expect(called).toBe(false);+ });++ test('should throw on goto timeouts (markBad session)', async () => {+ const baseUrl = 'https://example.com/';+ const requestList = await Apify.openRequestList(null, [baseUrl]);+ const gotoTimeoutSecs = 0.001;++ let called = false;+ const failedRequests = [];+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ useSessionPool: true,+ persistCookiesPerSession: false,+ maxRequestRetries: 0,+ gotoTimeoutSecs,+ handlePageFunction: async () => { // eslint-disable-line no-loop-func+ called = true;+ },+ handleFailedRequestFunction: async ({ request }) => {+ failedRequests.push(request);+ },+ });++ await crawler.run();++ expect(failedRequests.length).toBe(1);+ failedRequests.forEach((fr) => {+ const [msg] = fr.errorMessages;+ expect(msg).toContain(`gotoFunction timed out after ${gotoTimeoutSecs} seconds.`);+ });+ expect(called).toBe(false);+ });++ test('should throw on "blocked" status codes (retire session)', async () => {+ const baseUrl = 'https://example.com/';+ const sources = STATUS_CODES_BLOCKED.map((statusCode) => {+ return {+ url: baseUrl + statusCode,+ userData: { statusCode },+ };+ });+ const requestList = await Apify.openRequestList(null, sources);++ let called = false;+ const failedRequests = [];+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ useSessionPool: true,+ persistCookiesPerSession: false,+ maxRequestRetries: 0,+ gotoFunction: async ({ request }) => {+ return { status: () => request.userData.statusCode };+ },+ handlePageFunction: async () => { // eslint-disable-line no-loop-func+ called = true;+ },+ handleFailedRequestFunction: async ({ request }) => {+ failedRequests.push(request);+ },+ });++ await crawler.run();++ expect(failedRequests.length).toBe(STATUS_CODES_BLOCKED.length);+ failedRequests.forEach((fr) => {+ const [msg] = fr.errorMessages;+ expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);+ });+ expect(called).toBe(false);+ });+describe('proxy', () => {let requestList;beforeEach(async () => {