Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixes HTTP2 in requestAsBrowser (#866)
petrpatek537859e5267e
diff --git a/src/utils_request.js b/src/utils_request.jsindex cbe35774e..e09abc98c 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -105,6 +105,7 @@ export const requestAsBrowser = async (options) => {abortFunction,ignoreSslErrors = true,useInsecureHttpParser = true,+ useHttp2 = false,...otherParams} = options;@@ -113,7 +114,7 @@ export const requestAsBrowser = async (options) => {Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': `${languageCode}-${countryCode},${languageCode};q=0.5`,'Accept-Encoding': 'gzip, deflate, br',- Connection: 'keep-alive',+ Connection: useHttp2 ? undefined : 'keep-alive',};const requestOpts = {@@ -128,6 +129,7 @@ export const requestAsBrowser = async (options) => {abortFunction,ignoreSslErrors,insecureHTTPParser: useInsecureHttpParser,+ useHttp2,};try {diff --git a/test/utils_request.test.js b/test/utils_request.test.jsindex 88ed15cd2..8412bb8f4 100644--- a/test/utils_request.test.js+++ b/test/utils_request.test.js@@ -235,5 +235,10 @@ describe('Apify.utils_request', () => {if (err.name === 'MaxRedirectsError') throw err;}});++ test('works with useHttp2', async () => {+ const url = 'https://www.amazon.com/s?k=iphone';+ await requestAsBrowser({ url, useHttp2: true });+ });});});