Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix tests
Ondra Urban6b17f8987bb4
diff --git a/src/request.js b/src/request.jsindex c744e1f5c..21575b64e 100644--- a/src/request.js+++ b/src/request.js@@ -9,7 +9,7 @@ export function computeUniqueKey({ url, method, payload, keepUrlFragment }) {const normalizedMethod = method.toUpperCase();const normalizedUrl = normalizeUrl(url, keepUrlFragment);if (normalizedMethod === 'GET') return normalizedUrl;- const payloadHash = hashPayload(payload);+ const payloadHash = payload ? hashPayload(payload) : '';return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;}@@ -54,14 +54,14 @@ export function hashPayload(payload) {* @param {String} [options.uniqueKey] A unique key identifying the request.* Two requests with the same `uniqueKey` are considered as pointing to the same URL.*- * If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.+ * If `uniqueKey` is not provided, then for GET requests it is automatically generated by normalizing the URL.* For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`* of `http://www.example.com/something`.** The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.- * Beware that the HTTP method and payload are not included in the `uniqueKey`,- * so requests to the same URL but with different HTTP methods or different POST payloads- * are all considered equal and will be deduplicated by {@link RequestList} and {@link RequestQueue}.+ *+ * For other than GET requests, method and payload are included in the generated `uniqueKey`,+ * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`.** Pass an arbitrary non-empty text value to the `uniqueKey` property* to override the default behavior and specify which URLs shall be considered equal.@@ -101,7 +101,7 @@ export function hashPayload(payload) {* Two requests with the same `uniqueKey` are considered as pointing to the same URL.* @property {String} method* HTTP method, e.g. `GET` or `POST`.- * @property {String} payload+ * @property {String|Buffer} payload* HTTP request payload, e.g. for POST requests.* @property {Boolean} noRetry* The `true` value indicates that the request will not be automatically retried on error.