Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Finalize tests
Ondra Urbandda38ee1e242
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex caca192d8..ae57ff2cc 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -310,8 +310,8 @@ class BasicCrawler {// If we got here, it means we actually want to handle the error.request.pushErrorMessage(error);- // Reclaim and retry request if retryCount is not exceeded.- if (request.retryCount < this.maxRequestRetries) {+ // Reclaim and retry request if request is not skipped and retryCount is not exceeded.+ if (!request.skipped && request.retryCount < this.maxRequestRetries) {request.retryCount++;log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-lenurl: request.url,diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex a0f21f5a2..4a0784479 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -356,20 +356,25 @@ class CheerioCrawler {return reject(new Error(`${status} - ${message}`));}// It's not a JSON so it's probably some text. Get the first 100 chars of it.- return reject(new Error(`${status} - Internal Server Error: ${body.substr(0, 100)}`));+ return reject(new Error(`CheerioCrawler: ${status} - Internal Server Error: ${body.substr(0, 100)}`));}// Handle situations where the server explicitly states that// it will not serve the resource as text/html by skipping.if (status === 406) {- return log.error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);+ request.skip();+ return reject(new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`));}// Other 200-499 responses are considered OK, but first check the content type.if (type === 'text/html') {- resolve(body);+ res.body = body;+ resolve(res);} else {- log.error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`);+ request.skip();+ reject(new Error(+ `CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`,+ ));}}).on('error', err => reject(err));diff --git a/src/puppeteer.js b/src/puppeteer.jsindex a62858277..492ab8b38 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -163,7 +163,8 @@ const getPuppeteerOrThrow = () => {* <a href="https://www.apify.com/docs/actor#base-images" target="_blank">Apify Actor documentation</a>* for details.*- * For an example of usage, see the [Synchronous run Example](../examples/synchronousrun) or the [Puppeteer proxy Example](../examples/puppeteerwithproxy)+ * For an example of usage, see the [Synchronous run Example](../examples/synchronousrun)+ * or the [Puppeteer proxy Example](../examples/puppeteerwithproxy)** @param {LaunchPuppeteerOptions} [options]* Optional settings passed to `puppeteer.launch()`. In addition todiff --git a/src/request.js b/src/request.jsindex 65be21137..0088add30 100644--- a/src/request.js+++ b/src/request.js@@ -144,6 +144,7 @@ class Request {this.userData = userData;this.ignoreErrors = ignoreErrors;this.handledAt = handledAt && new Date(handledAt);+ this.skipped = false;}/**@@ -192,6 +193,15 @@ class Request {if (!this.errorMessages) this.errorMessages = [];this.errorMessages.push(message);}++ /**+ * Marks the request as skipped which in turn prevents {@link BasicCrawler}+ * (and {@PuppeteerCrawler} + {@CheerioCrawler}, since they use {@BasicCrawler} internally)+ * from retrying the request after an error occurs.+ */+ skip() {+ this.skipped = true;+ }}export default Request;diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 4fa86e678..8d1b08fe3 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -298,66 +298,6 @@ describe('CheerioCrawler', () => {headers.forEach(h => expect(h.Accept).to.be.eql('text/html'));});- describe('by skipping', () => {- let crawler;- let handlePageInvocationCount = 0;- let handleFailInvocationCount = 0;- let logMessages = [];- let origLogError;- beforeEach(() => {- crawler = new Apify.CheerioCrawler({- requestList,- maxRequestRetries: 1,- handlePageFunction: async () => {- handlePageInvocationCount++;- },- handleFailedRequestFunction: async () => {- handleFailInvocationCount++;- },- });- origLogError = log.error;- log.error = arg => logMessages.push(arg);- });- afterEach(async () => {- crawler = null;- handlePageInvocationCount = 0;- logMessages = [];- log.error = origLogError;- origLogError = null;- });-- it('when 406 is received', async () => {- // Mock Request to respond with a 406.- crawler.rqst = () => {- const response = new Readable({- read() {- this.push('x');- this.push(null);- },- });- response.headers = {- 'content-type': 'text/plain',- };- response.statusCode = 406;-- const ee = new EventEmitter();-- setTimeout(() => {- ee.emit('response', response);- }, 0);-- return ee;- };-- await crawler.run();-- expect(handlePageInvocationCount).to.be.eql(4);- expect(handleFailInvocationCount).to.be.eql(0);- expect(logMessages).to.have.lengthOf(4);- logMessages.forEach(msg => expect(msg).to.include('is not available in HTML format'));- });- });-describe('by throwing', () => {let crawler;let handlePageInvocationCount = 0;@@ -534,6 +474,66 @@ describe('CheerioCrawler', () => {expect(errorMessages).to.have.lengthOf(8);errorMessages.forEach(msg => expect(msg).to.include('500 - Hello'));});++ it('when 406 is received', async () => {+ // Mock Request to respond with a 406.+ crawler.rqst = () => {+ const response = new Readable({+ read() {+ this.push('x');+ this.push(null);+ },+ });+ response.headers = {+ 'content-type': 'text/plain',+ };+ response.statusCode = 406;++ const ee = new EventEmitter();++ setTimeout(() => {+ ee.emit('response', response);+ }, 0);++ return ee;+ };++ await crawler.run();++ expect(handlePageInvocationCount).to.be.eql(0);+ expect(errorMessages).to.have.lengthOf(4);+ errorMessages.forEach(msg => expect(msg).to.include('is not available in HTML format'));+ });++ it('when status is ok, but a wrong content type is received', async () => {+ // Mock Request to respond with a 406.+ crawler.rqst = () => {+ const response = new Readable({+ read() {+ this.push('x');+ this.push(null);+ },+ });+ response.headers = {+ 'content-type': 'application/json',+ };+ response.statusCode = 200;++ const ee = new EventEmitter();++ setTimeout(() => {+ ee.emit('response', response);+ }, 0);++ return ee;+ };++ await crawler.run();++ expect(handlePageInvocationCount).to.be.eql(0);+ expect(errorMessages).to.have.lengthOf(4);+ errorMessages.forEach(msg => expect(msg).to.include('served Content-Type application/json instead of text/html'));+ });});});