Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Rewrote to use the http request.
Petr Patek1b16f07bd0bc
diff --git a/package.json b/package.jsonindex 1b48bc078..b44bf6082 100644--- a/package.json+++ b/package.json@@ -49,6 +49,7 @@"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"},"dependencies": {+ "@apify/http-request": "^1.0.0","@apify/ps-tree": "^1.1.3","apify-client": "^0.5.13","apify-shared": "^0.1.41",diff --git a/src/utils_request.js b/src/utils_request.jsindex 0690b0b3f..71a707f96 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -1,7 +1,7 @@import contentType from 'content-type';import * as url from 'url';import _ from 'underscore';-import httpRequest from 'http-request';+import httpRequest from '@apify/http-request';const FIREFOX_MOBILE_USER_AGENT = 'Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0';const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko)'diff --git a/test/utils_request.js b/test/utils_request.jsindex a88120d5c..dc0b34801 100644--- a/test/utils_request.js+++ b/test/utils_request.js@@ -6,7 +6,6 @@ import { startExpressAppPromise } from './_helper';const CONTENT = 'CONTENT';const HOST = '127.0.0.1';-const ERROR_BODY = 'CUSTOM_ERROR';describe('Apify.utils_request', () => {let mochaListener;@@ -160,15 +159,5 @@ describe('Apify.utils_request', () => {expect(response.statusCode).to.eql(200);expect(JSON.parse(response.body)['user-agent']).to.be.eql(options.headers['User-Agent']);});-- it('overrides defaults', async () => {- const options = {- url: 'https://www.ebay.com/sch/sis.html?_nkw=Beechcraft',- };-- const response = await requestAsBrowser(options);-- console.log(response);- });});});