Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix requestAsBrowser max redirects error

Ondra Urban74bcb40500f2
diff --git a/package.json b/package.json
index 310533b7a..014451d9f 100644
--- a/package.json
+++ b/package.json
@@ -51,7 +51,7 @@
"lint:fix": "eslint ./src ./test --ext .js,.jsx --fix"
},
"dependencies": {
- "@apify/http-request": "^1.2.0",
+ "@apify/http-request": "^2.0.0",
"@apify/ps-tree": "^1.1.3",
"@types/cheerio": "^0.22.18",
"@types/node": "^12",
diff --git a/src/utils_request.js b/src/utils_request.js
index 796297e9b..be4e28d65 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -107,10 +107,7 @@ export const requestAsBrowser = async (options) => {
...otherParams
} = options;
- const parsedUrl = new URL(url);
-
const defaultHeaders = {
- Host: parsedUrl.host,
'User-Agent': useMobileVersion ? FIREFOX_MOBILE_USER_AGENT : FIREFOX_DESKTOP_USER_AGENT,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': `${languageCode}-${countryCode},${languageCode};q=0.5`,
diff --git a/test/utils_request.test.js b/test/utils_request.test.js
index 3484b1b91..dbc63746f 100644
--- a/test/utils_request.test.js
+++ b/test/utils_request.test.js
@@ -225,5 +225,16 @@ describe('Apify.utils_request', () => {
}
}
});
+
+ test('does not get into redirect loops', async () => {
+ const url = 'https://www.smartmania.cz'; // uses www to no-www redirect
+ try {
+ await requestAsBrowser({ url });
+ } catch (err) {
+ // If it's some other error, it's fine for the purpose of this test.
+ // We're only making sure that the max redirect error is not there.
+ if (err.name === 'MaxRedirectsError') throw err;
+ }
+ });
});
});