Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Rework merging of headers in requestAsBrowser

Ondra Urbanb01a15f6c241
diff --git a/docs/api/utils.md b/docs/api/utils.md
index 8f62ca0ee..cf8107c11 100644
--- a/docs/api/utils.md
+++ b/docs/api/utils.md
@@ -20,7 +20,7 @@ await Apify.utils.sleep(1500);
- [`utils`](#utils) : `object`
- [`.enqueueLinks`](#utils.enqueueLinks) ⇒ `Promise<Array<QueueOperationInfo>>`
- - [`.requestAsBrowser`](#utils.requestAsBrowser) ⇒ `Promise<(http.IncomingMesage|stream.Readable)>`
+ - [`.requestAsBrowser`](#utils.requestAsBrowser) ⇒ `Promise<(http.IncomingMessage|stream.Readable)>`
- [`.sleep`](#utils.sleep) ⇒ `Promise`
- [`.URL_NO_COMMAS_REGEX`](#utils.URL_NO_COMMAS_REGEX)
- [`.URL_WITH_COMMAS_REGEX`](#utils.URL_WITH_COMMAS_REGEX)
@@ -137,7 +137,7 @@ objects.
</table>
<a name="utils.requestAsBrowser"></a>
-## `utils.requestAsBrowser` ⇒ `Promise<(http.IncomingMesage|stream.Readable)>`
+## `utils.requestAsBrowser` ⇒ `Promise<(http.IncomingMessage|stream.Readable)>`
Sends a HTTP request that looks like a request sent by a web browser, fully emulating browser's HTTP headers.
diff --git a/src/utils_request.js b/src/utils_request.js
index 6d6f440b6..d0fa0f80c 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -64,7 +64,7 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
* If function returns true request gets aborted. This function is passed to the
* (@apify/http-request)[https://www.npmjs.com/package/@apify/http-request] NPM package.
*
- * @return {Promise<http.IncomingMesage|stream.Readable>}
+ * @return {Promise<http.IncomingMessage|stream.Readable>}
* @memberOf utils
* @name requestAsBrowser
*/
@@ -81,8 +81,9 @@ export const requestAsBrowser = async (options) => {
'Accept-Encoding': 'gzip, deflate, br',
Connection: 'keep-alive',
};
- opts.headers = _.defaults({}, opts.headers, defaultHeaders);
-
+ // Users can provide headers in lowercase so we need to make sure
+ // that their values are applied, but names are kept upper-case.
+ opts.headers = mergeHeaders(opts.headers, defaultHeaders);
try {
return await httpRequest(opts);
@@ -94,3 +95,17 @@ export const requestAsBrowser = async (options) => {
throw e;
}
};
+
+function mergeHeaders(userHeaders, defaultHeaders) {
+ const headers = { ...defaultHeaders, ...userHeaders };
+ Object.keys(headers).forEach((key) => {
+ const lowerCaseKey = key.toLowerCase();
+ const keyIsNotLowerCase = key !== lowerCaseKey;
+ // eslint-disable-next-line
+ if (keyIsNotLowerCase && headers.hasOwnProperty(lowerCaseKey)) {
+ headers[key] = headers[lowerCaseKey];
+ delete headers[lowerCaseKey];
+ }
+ });
+ return headers;
+}
diff --git a/test/utils_request.test.js b/test/utils_request.test.js
index 6f31fe788..f51c6b434 100644
--- a/test/utils_request.test.js
+++ b/test/utils_request.test.js
@@ -186,5 +186,24 @@ describe('Apify.utils_request', () => {
expect(headersArray[10]).toBe('Connection');
expect(headersArray[11]).toBe('keep-alive');
});
+
+ test('custom headers in lowercase override uppercase defaults', async () => {
+ const host = `${HOST}:${port}`;
+ const options = {
+ url: `http://${host}/rawHeaders`,
+ headers: {
+ accept: 'foo',
+ bar: 'BAZ',
+ },
+ };
+
+ const response = await requestAsBrowser(options);
+ const headersArray = JSON.parse(response.body);
+ expect(response.statusCode).toBe(200);
+ expect(headersArray[4]).toBe('Accept');
+ expect(headersArray[5]).toBe('foo');
+ expect(headersArray[12]).toBe('bar');
+ expect(headersArray[13]).toBe('BAZ');
+ });
});
});