Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added httpRequest v1.0.1 support.
Petr Patek617d5fc71dbe
diff --git a/package.json b/package.jsonindex 44acf58b5..fd25212df 100644--- a/package.json+++ b/package.json@@ -50,7 +50,7 @@"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"},"dependencies": {- "@apify/http-request": "^1.0.0",+ "@apify/http-request": "^1.0.1","@apify/ps-tree": "^1.1.3","apify-client": "^0.5.22","apify-shared": "^0.1.51",diff --git a/src/utils_request.js b/src/utils_request.jsindex c24d245c0..232be7b59 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -18,6 +18,7 @@ export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {const { type } = contentType.parse(res.headers['content-type']);return res.statusCode === 406 || type.toLowerCase() !== 'text/html';},+ useCaseSensitiveHeaders: true,};/*** Sends a HTTP request that looks like a request sent by a web browser,diff --git a/test/utils_request.js b/test/utils_request.jsindex 7716cefb1..a8cfa4cdc 100644--- a/test/utils_request.js+++ b/test/utils_request.js@@ -1,6 +1,11 @@import { expect } from 'chai';import express from 'express';-import { requestAsBrowser, REQUEST_AS_BROWSER_DEFAULT_OPTIONS, FIREFOX_MOBILE_USER_AGENT, FIREFOX_DESKTOP_USER_AGENT } from '../build/utils_request';+import {+ requestAsBrowser,+ REQUEST_AS_BROWSER_DEFAULT_OPTIONS,+ FIREFOX_MOBILE_USER_AGENT,+ FIREFOX_DESKTOP_USER_AGENT,+} from '../build/utils_request';import { startExpressAppPromise } from './_helper';const CONTENT = 'CONTENT';@@ -161,7 +166,7 @@ describe('Apify.utils_request', () => {expect(JSON.parse(response.body)['user-agent']).to.be.eql(options.headers['User-Agent']);});- it('headers has same order as in Firefox', async () => {+ it('headers has same format as in firefox', async () => {const host = `${HOST}:${port}`;const options = {url: `http://${host}/rawHeaders`,@@ -169,19 +174,20 @@ describe('Apify.utils_request', () => {const response = await requestAsBrowser(options);const headersArray = JSON.parse(response.body);-expect(response.statusCode).to.eql(200);- expect(headersArray[0]).to.be.eql('user-agent');+++ expect(headersArray[0]).to.be.eql('User-Agent');expect(headersArray[1]).to.be.eql(FIREFOX_DESKTOP_USER_AGENT);- expect(headersArray[2]).to.be.eql('host');+ expect(headersArray[2]).to.be.eql('Host');expect(headersArray[3]).to.be.eql(host);- expect(headersArray[4]).to.be.eql('accept');+ expect(headersArray[4]).to.be.eql('Accept');expect(headersArray[5]).to.be.eql('text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8');- expect(headersArray[6]).to.be.eql('accept-language');+ expect(headersArray[6]).to.be.eql('Accept-Language');expect(headersArray[7]).to.be.eql('en-US,en;q=0.5');- expect(headersArray[8]).to.be.eql('accept-encoding');+ expect(headersArray[8]).to.be.eql('Accept-Encoding');expect(headersArray[9]).to.be.eql('gzip, deflate, br');- expect(headersArray[10]).to.be.eql('connection');+ expect(headersArray[10]).to.be.eql('Connection');expect(headersArray[11]).to.be.eql('keep-alive');});});