Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Deprecate request.doNotRetry; update uniqueKey generation
Ondra Urban6cd27fede6de
diff --git a/docs/api/Request.md b/docs/api/Request.mdindex dd98f518a..5a0e68a60 100644--- a/docs/api/Request.md+++ b/docs/api/Request.md@@ -110,7 +110,7 @@ const foo = request.userData.foo;* [Request](request)* [`new Request(options)`](#new_Request_new)* [`.pushErrorMessage(errorOrMessage, [options])`](#Request+pushErrorMessage)- * [`.doNotRetry([message])`](#Request+doNotRetry)+ * ~~[`.doNotRetry([message])`](#Request+doNotRetry)~~<a name="new_Request_new"></a>@@ -218,7 +218,9 @@ as possible, since just throwing a bad type error makes any debugging rather dif</table><a name="Request+doNotRetry"></a>-## `request.doNotRetry([message])`+## ~~`request.doNotRetry([message])`~~+***Deprecated***+Flags the request with no retry which prevents [`BasicCrawler`](basiccrawler)(as well as {@PuppeteerCrawler} and {@CheerioCrawler}, since they use {@BasicCrawler} internally)from retrying the request after an error occurs.diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 3efd61bcd..d5d92bedc 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -412,14 +412,14 @@ class CheerioCrawler {// Handle situations where the server explicitly states that// it will not serve the resource as text/html by skipping.if (status === 406) {- request.doNotRetry();+ request.noRetry = true;response.destroy();return reject(new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`));}// Other 200-499 responses are considered OK, but first check the content type.if (type !== 'text/html') {- request.doNotRetry();+ request.noRetry = true;response.destroy();return reject(new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`,diff --git a/src/request.js b/src/request.jsindex 310b58ec5..c744e1f5c 100644--- a/src/request.js+++ b/src/request.js@@ -1,9 +1,26 @@import _ from 'underscore';import util from 'util';+import crypto from 'crypto';import { checkParamOrThrow } from 'apify-client/build/utils';import { normalizeUrl } from 'apify-shared/utilities';+import log from 'apify-shared/log';-export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keepUrlFragment);+export function computeUniqueKey({ url, method, payload, keepUrlFragment }) {+ const normalizedMethod = method.toUpperCase();+ const normalizedUrl = normalizeUrl(url, keepUrlFragment);+ if (normalizedMethod === 'GET') return normalizedUrl;+ const payloadHash = hashPayload(payload);+ return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;+}++export function hashPayload(payload) {+ return crypto+ .createHash('sha256')+ .update(payload)+ .digest('base64')+ .replace(/(\+|\/|=)/g, '')+ .substr(0, 8);+}/*** Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.@@ -142,7 +159,7 @@ class Request {this.url = url;this.loadedUrl = loadedUrl;// NOTE: If URL is invalid, computeUniqueKey() returns null which was causing weird errors- this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment) || url;+ this.uniqueKey = uniqueKey || computeUniqueKey({ url, method, payload, keepUrlFragment }) || url;this.method = method;this.payload = payload;this.noRetry = noRetry;@@ -151,7 +168,8 @@ class Request {this.headers = headers;this.userData = userData;- // TODO: What is this text parsing good for? There's no unit test for it...+ // Requests received from API will have ISOString dates,+ // but we want to have a Date instance.// eslint-disable-next-line no-nested-ternarythis.handledAt = _.isDate(handledAt)? handledAt@@ -214,11 +232,11 @@ class Request {* and throw an Error.** @param {String} [message]+ * @deprecated 2019/06/26*/doNotRetry(message) {+ log.deprecated('request.doNotRetry is deprecated. Use request.noRetry = true; instead.');this.noRetry = true;- // TODO (JC 2019-05-03): Frankly, I think this function shouldn't be in the SDK, it has zero value really- // and only adds a confusion and legacy burden. I'd just deprecate it.if (message) throw new Error(message);}}diff --git a/test/request.js b/test/request.jsindex 276b4a8ea..d2b9d4dae 100644--- a/test/request.js+++ b/test/request.js@@ -1,6 +1,7 @@import util from 'util';import { expect } from 'chai';-import { computeUniqueKey } from '../build/request';+import { normalizeUrl } from 'apify-shared/utilities';+import { hashPayload } from '../build/request';import Apify from '../build/index';describe('Apify.Request', () => {@@ -9,13 +10,25 @@ describe('Apify.Request', () => {expect(() => new Apify.Request({ url: 'xxx' })).to.not.throw();});- it('should create unique based on url', () => {+ it('should create unique key based on url for GET requests', () => {const url = 'https://user:[email protected]/a/vb/c /d?q=1&q=kjnjkn$lkn#lkmlkml';- const normalizedUrl = computeUniqueKey(url);+ const normalizedUrl = normalizeUrl(url);const request = new Apify.Request({ url });expect(request.uniqueKey).to.be.eql(normalizedUrl);- expect(normalizedUrl).to.not.eql(url);+ expect(request.uniqueKey).to.not.eql(request.url);+ });++ it('should create unique key based on url, method and payload for POST requests', () => {+ const url = 'https://user:[email protected]/a/vb/c /d?q=1&q=kjnjkn$lkn#lkmlkml';+ const payload = JSON.stringify({ foo: 'bar' });+ const payloadHash = hashPayload(payload);+ const normalizedUrl = normalizeUrl(url);+ const request = new Apify.Request({ url, method: 'post', payload });++ const uniqueKey = `POST(${payloadHash}):${normalizedUrl}`;++ expect(request.uniqueKey).to.be.eql(uniqueKey);});it('works', () => {@@ -39,6 +52,9 @@ describe('Apify.Request', () => {handledAt: new Date(),};expect(new Apify.Request(data)).to.include(data);++ data.handledAt = (new Date()).toISOString();+ expect((new Apify.Request(data)).handledAt).to.be.a('date');});it('should allow to push error messages', () => {@@ -97,7 +113,7 @@ describe('Apify.Request', () => {expect(last).to.include(__filename.split(/[\\/]/g).pop());});- it('should should allow to have a GET request with payload', () => {+ it('should not allow to have a GET request with payload', () => {expect(() => new Apify.Request({ url: 'http://example.com', payload: 'foo' })).to.throw();expect(() => new Apify.Request({ url: 'http://example.com', payload: 'foo', method: 'POST' })).to.not.throw();});