Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Renamed the checkStatus function to retireOnStatusCode and allow of custom status codes.
Petr Patek384b04cf4b0e
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 629d359b2..9f8d07a91 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -613,7 +613,7 @@ class CheerioCrawler {* @private*/_throwOnBlockedRequest(session, statusCode) {- const isBlocked = session.checkStatus(statusCode);+ const isBlocked = session.retireOnBlockedStatusCodes(statusCode);if (isBlocked) {throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex 207a503d4..1c720ea75 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -184,10 +184,11 @@ export class Session {/*** Retires session based on status code.* @param statusCode {Number} - HTTP status code+ * @param blockedStatusCodes {Array<Number>} - Custom HTTP status codes that means blocking on particular website.* @return {boolean} whether the session was retired.*/- checkStatus(statusCode) {- const isBlocked = STATUS_CODES_BLOCKED.includes(statusCode);+ retireOnBlockedStatusCodes(statusCode, blockedStatusCodes = []) {+ const isBlocked = STATUS_CODES_BLOCKED.concat(blockedStatusCodes).includes(statusCode);if (isBlocked) {this.retire();}diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex 15b622e75..d78379e0a 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -41,7 +41,7 @@ describe('Session - testing session behaviour ', () => {},);- test('should mark session markBaded', () => {+ test('should mark session markBad', () => {session.markBad();expect(session.errorScore).toBe(1);expect(session.usageCount).toBe(1);@@ -129,20 +129,31 @@ describe('Session - testing session behaviour ', () => {test('should checkStatus work', () => {session = new Session({ sessionPool });- expect(session.checkStatus(100)).toBeFalsy();- expect(session.checkStatus(200)).toBeFalsy();- expect(session.checkStatus(400)).toBeFalsy();- expect(session.checkStatus(500)).toBeFalsy();+ expect(session.retireOnBlockedStatusCodes(100)).toBeFalsy();+ expect(session.retireOnBlockedStatusCodes(200)).toBeFalsy();+ expect(session.retireOnBlockedStatusCodes(400)).toBeFalsy();+ expect(session.retireOnBlockedStatusCodes(500)).toBeFalsy();STATUS_CODES_BLOCKED.forEach((status) => {const sess = new Session({ sessionPool });let isCalled;const call = () => { isCalled = true; };sess.retire = call;- expect(sess.checkStatus(status)).toBeTruthy();+ expect(sess.retireOnBlockedStatusCodes(status)).toBeTruthy();expect(isCalled).toBeTruthy();});});+ test('should checkStatus work with custom codes', () => {+ session = new Session({ sessionPool });+ const customStatusCodes = [100, 202, 300];+ expect(session.retireOnBlockedStatusCodes(100, customStatusCodes)).toBeTruthy();+ expect(session.retireOnBlockedStatusCodes(101, customStatusCodes)).toBeFalsy();+ expect(session.retireOnBlockedStatusCodes(200, customStatusCodes)).toBeFalsy();+ expect(session.retireOnBlockedStatusCodes(202, customStatusCodes)).toBeTruthy();+ expect(session.retireOnBlockedStatusCodes(300, customStatusCodes)).toBeTruthy();+ expect(session.retireOnBlockedStatusCodes(400, customStatusCodes)).toBeFalsy();+ });+describe('.putResponse & .getCookieString', () => {test('should set and update cookies from "set-cookie" header', () => {const headers = {};