Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Remove remnants of request package (#608)

Ondra Urbana5e360dda78d
diff --git a/docs/api/Apify.md b/docs/api/Apify.md
index 0cf09965c..9dfd7461e 100644
--- a/docs/api/Apify.md
+++ b/docs/api/Apify.md
@@ -479,7 +479,7 @@ Apify.main(() => {
If the user function returns a promise, it is considered asynchronous:
```javascript
-const request = require('request-promise-native');
+const { requestAsBrowser } = require('some-request-library');
Apify.main(() => {
// My asynchronous function that returns a promise
@@ -492,7 +492,7 @@ Apify.main(() => {
To simplify your code, you can take advantage of the `async`/`await` keywords:
```javascript
-const request = require('request-promise-native');
+const request = require('some-request-library');
Apify.main(async () => {
// My asynchronous function
diff --git a/docs/api/BasicCrawler.md b/docs/api/BasicCrawler.md
index 9350ad0d3..4b538192b 100644
--- a/docs/api/BasicCrawler.md
+++ b/docs/api/BasicCrawler.md
@@ -33,8 +33,6 @@ New requests are only dispatched when there is enough free CPU and memory availa
**Example usage:**
```javascript
-const rp = require('request-promise-native');
-
// Prepare a list of URLs to crawl
const requestList = new Apify.RequestList({
sources: [{ url: 'http://www.example.com/page-1' }, { url: 'http://www.example.com/page-2' }],
@@ -47,9 +45,10 @@ const crawler = new Apify.BasicCrawler({
handleRequestFunction: async ({ request }) => {
// 'request' contains an instance of the Request class
// Here we simply fetch the HTML of the page and store it to a dataset
+ const { body } = await Apify.utils.requestAsBrowser(request);
await Apify.pushData({
url: request.url,
- html: await rp(request.url),
+ html: body,
});
},
});
diff --git a/docs/examples/BasicCrawler.md b/docs/examples/BasicCrawler.md
index c41d1b532..88cd6b850 100644
--- a/docs/examples/BasicCrawler.md
+++ b/docs/examples/BasicCrawler.md
@@ -5,15 +5,14 @@ title: Basic Crawler
This is the most basic example of the Apify SDK, which demonstrates some of its elementary tools, such as the
[`BasicCrawler`](/docs/api/basic-crawler) and [`RequestList`](/docs/api/request-list) classes. The script just downloads several web pages with plain
-HTTP requests (using the [`request-promise`](https://www.npmjs.com/package/request-promise) library) and stores their raw HTML and URL to the default
-dataset. In local configuration, the data will be stored as JSON files in `./apify_storage/datasets/default`.
+HTTP requests using the [`Apify.utils.requestAsBrowser()`](/docs/api/utils#requestasbrowser) convenience function and stores their raw HTML and URL to
+the default dataset. In local configuration, the data will be stored as JSON files in `./apify_storage/datasets/default`.
To run this example on the Apify Platform, select the `Node.js 12 on Alpine Linux (apify/actor-node-basic)` base image on the source tab of your actor
configuration.
```javascript
const Apify = require('apify');
-const requestPromise = require('request-promise');
// Apify.main() function wraps the crawler logic (it is optional).
Apify.main(async () => {
@@ -42,12 +41,12 @@ Apify.main(async () => {
console.log(`Processing ${request.url}...`);
// Fetch the page HTML
- const html = await requestPromise(request.url);
+ const { body } = await Apify.utils.requestAsBrowser(request);
// Store the HTML and URL to the default dataset.
await Apify.pushData({
url: request.url,
- html,
+ html: body,
});
},
});
diff --git a/examples/basic_crawler.js b/examples/basic_crawler.js
index f34283c97..ec5f33dec 100644
--- a/examples/basic_crawler.js
+++ b/examples/basic_crawler.js
@@ -3,9 +3,9 @@
* elementary tools, such as the
* [`BasicCrawler`](/docs/api/basic-crawler)
* and [`RequestList`](/docs/api/request-list) classes.
- * The script just downloads several web pages with plain HTTP requests (using the
- * [`request-promise`](https://www.npmjs.com/package/request-promise) library)
- * and stores their raw HTML and URL to the default dataset.
+ * The script just downloads several web pages with plain HTTP requests using the
+ * [`Apify.utils.requestAsBrowser()`](/docs/api/utils#requestasbrowser)
+ * convenience function and stores their raw HTML and URL to the default dataset.
* In local configuration, the data will be stored as JSON files in `./apify_storage/datasets/default`.
*
* To run this example on the Apify Platform, select the `Node.js 12 on Alpine Linux (apify/actor-node-basic)` base image
@@ -13,7 +13,6 @@
*/
const Apify = require('apify');
-const requestPromise = require('request-promise');
// Apify.main() function wraps the crawler logic (it is optional).
Apify.main(async () => {
@@ -43,12 +42,12 @@ Apify.main(async () => {
console.log(`Processing ${request.url}...`);
// Fetch the page HTML
- const html = await requestPromise(request.url);
+ const { body } = await Apify.utils.requestAsBrowser(request);
// Store the HTML and URL to the default dataset.
await Apify.pushData({
url: request.url,
- html,
+ html: body,
});
},
});
diff --git a/package.json b/package.json
index 8c513bb1c..4815342a9 100644
--- a/package.json
+++ b/package.json
@@ -69,8 +69,6 @@
"jquery": "^3.4.1",
"mime-types": "^2.1.24",
"proxy-chain": "^0.4.1",
- "request": "^2.88.0",
- "request-promise-native": "^1.0.5",
"rimraf": "^3.0.2",
"semver": "^7.1.3",
"socket.io": "^2.2.0",
diff --git a/src/actor.js b/src/actor.js
index 2ac193f37..44b3efc5c 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -195,7 +195,7 @@ export const getEnv = () => {
*
* If the user function returns a promise, it is considered asynchronous:
* ```javascript
- * const request = require('request-promise-native');
+ * const { requestAsBrowser } = require('some-request-library');
*
* Apify.main(() => {
* // My asynchronous function that returns a promise
@@ -208,7 +208,7 @@ export const getEnv = () => {
* To simplify your code, you can take advantage of the `async`/`await` keywords:
*
* ```javascript
- * const request = require('request-promise-native');
+ * const request = require('some-request-library');
*
* Apify.main(async () => {
* // My asynchronous function
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 6aee9a9c4..bde1f92b7 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -148,8 +148,6 @@ const DEFAULT_OPTIONS = {
* **Example usage:**
*
* ```javascript
- * const rp = require('request-promise-native');
- *
* // Prepare a list of URLs to crawl
* const requestList = new Apify.RequestList({
* sources: [
@@ -165,9 +163,10 @@ const DEFAULT_OPTIONS = {
* handleRequestFunction: async ({ request }) => {
* // 'request' contains an instance of the Request class
* // Here we simply fetch the HTML of the page and store it to a dataset
+ * const { body } = await Apify.utils.requestAsBrowser(request);
* await Apify.pushData({
* url: request.url,
- * html: await rp(request.url),
+ * html: body,
* })
* },
* });
diff --git a/src/utils.js b/src/utils.js
index 725dc0fae..85ccde06a 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -11,12 +11,12 @@ import * as fsExtra from 'fs-extra';
import * as mime from 'mime-types';
import * as os from 'os';
import * as path from 'path';
-import * as requestPromise from 'request-promise-native';
import * as semver from 'semver';
import * as _ from 'underscore';
import { URL } from 'url';
import * as util from 'util';
import { USER_AGENT_LIST } from './constants';
+import * as requestUtils from './utils_request';
import log from './utils_log';
import { version as apifyVersion } from '../package.json';
@@ -412,8 +412,9 @@ const downloadListOfUrls = async ({ url, encoding = 'utf8', urlRegExp = URL_NO_C
checkParamOrThrow(url, 'url', 'String');
checkParamOrThrow(encoding, 'string', 'String');
checkParamOrThrow(urlRegExp, 'urlRegExp', 'RegExp');
+ const { requestAsBrowser } = requestUtils;
- const string = await requestPromise.get({ url, encoding });
+ const { body: string } = await requestAsBrowser({ url, encoding });
return extractUrls({ string, urlRegExp });
};
diff --git a/test/autoscaling/snapshotter.test.js b/test/autoscaling/snapshotter.test.js
index ca0606987..84762e532 100644
--- a/test/autoscaling/snapshotter.test.js
+++ b/test/autoscaling/snapshotter.test.js
@@ -90,9 +90,9 @@ describe('Snapshotter', () => {
const eventLoopSnapshots = snapshotter.getEventLoopSample();
const cpuSnapshots = snapshotter.getCpuSample();
- expect(cpuSnapshots.length).toBeGreaterThan(2);
- expect(memorySnapshots.length).toBeGreaterThan(2);
- expect(eventLoopSnapshots.length).toBeGreaterThan(4);
+ expect(cpuSnapshots.length).toBeGreaterThanOrEqual(2);
+ expect(memorySnapshots.length).toBeGreaterThanOrEqual(2);
+ expect(eventLoopSnapshots.length).toBeGreaterThanOrEqual(4);
});
test('correctly marks CPU overloaded using Platform event', async () => {
diff --git a/test/live_view/live_view_server.test.js b/test/live_view/live_view_server.test.js
index 36edd7a04..8fb8f5498 100644
--- a/test/live_view/live_view_server.test.js
+++ b/test/live_view/live_view_server.test.js
@@ -1,12 +1,12 @@
import path from 'path';
import { promisify } from 'util';
-import rqst from 'request-promise-native';
import fs from 'fs-extra';
import io from 'socket.io-client';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import Apify from '../../build/index';
import { LOCAL_STORAGE_DIR } from '../_helper';
import LiveViewServer from '../../build/live_view/live_view_server';
+import { requestAsBrowser } from '../../build/utils_request';
const { utils: { log } } = Apify;
const emptyDir = promisify(fs.emptyDir);
@@ -106,12 +106,12 @@ describe('LiveViewServer', () => {
test('should return screenshots', async () => {
const snapshot0 = new Promise(resolve => socket.on('snapshot', resolve));
await lvs.serve(fakePage);
- const response0 = await rqst(`${BASE_URL}/screenshot/${((await snapshot0).screenshotIndex)}`);
- expect(response0).toBe('screenshot0');
+ const response0 = await requestAsBrowser({ url: `${BASE_URL}/screenshot/${((await snapshot0).screenshotIndex)}` });
+ expect(response0.body).toBe('screenshot0');
const snapshot1 = new Promise(resolve => socket.on('snapshot', resolve));
await lvs.serve(fakePage);
- const response1 = await rqst(`${BASE_URL}/screenshot/${(await snapshot1).screenshotIndex}`);
- expect(response1).toBe('screenshot1');
+ const response1 = await requestAsBrowser({ url: `${BASE_URL}/screenshot/${(await snapshot1).screenshotIndex}` });
+ expect(response1.body).toBe('screenshot1');
});
test('should not store more than maxScreenshotFiles screenshots', async () => {
diff --git a/test/request_list.test.js b/test/request_list.test.js
index 3ca7c4e1c..082d8a28f 100644
--- a/test/request_list.test.js
+++ b/test/request_list.test.js
@@ -1,12 +1,12 @@
import _ from 'underscore';
-import request from 'request-promise-native';
import sinon from 'sinon';
import log from '../build/utils_log';
import { ACTOR_EVENT_NAMES_EX } from '../build/constants';
+import { deserializeArray } from '../build/serialization';
import Apify from '../build/index';
import * as keyValueStore from '../build/key_value_store';
import * as utils from '../build/utils';
-import { deserializeArray } from '../build/serialization';
+import * as requestUtils from '../build/utils_request';
describe('Apify.RequestList', () => {
let ll;
@@ -154,14 +154,14 @@ describe('Apify.RequestList', () => {
);
test('should use regex parameter to parse urls', async () => {
- const mock = sinon.mock(request);
+ const mock = sinon.mock(requestUtils);
const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd';
const listArr = ['https://example.com', 'HTTP://google.com'];
- mock.expects('get')
+ mock.expects('requestAsBrowser')
.once()
.withArgs({ url: 'http://example.com/list-1', encoding: 'utf8' })
- .returns(Promise.resolve(listStr));
+ .resolves({ body: listStr });
const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g;
const requestList = new Apify.RequestList({
diff --git a/test/utils.test.js b/test/utils.test.js
index f0968466d..acbed7ff4 100644
--- a/test/utils.test.js
+++ b/test/utils.test.js
@@ -5,12 +5,12 @@ import path from 'path';
import os from 'os';
import cheerio from 'cheerio';
import semver from 'semver';
-import requestPromise from 'request-promise-native';
import LruCache from 'apify-shared/lru_cache';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
-import log from '../build/utils_log';
-import * as utils from '../build/utils';
import Apify from '../build/index';
+import * as utils from '../build/utils';
+import log from '../build/utils_log';
+import * as requestUtils from '../build/utils_request';
describe('utils.newClient()', () => {
test('reads environment variables correctly', () => {
@@ -505,16 +505,16 @@ describe('Apify.utils.downloadListOfUrls()', () => {
const { downloadListOfUrls } = utils.publicUtils;
let stub;
beforeEach(() => {
- stub = sinon.stub(requestPromise, 'get');
+ stub = sinon.stub(requestUtils, 'requestAsBrowser');
});
afterEach(() => {
- requestPromise.get.restore();
+ requestUtils.requestAsBrowser.restore();
});
test('downloads a list of URLs', () => {
const text = fs.readFileSync(path.join(__dirname, 'data', 'simple_url_list.txt'), 'utf8');
const arr = text.trim().split(/[\r\n]+/g).map(u => u.trim());
- stub.resolves(text);
+ stub.resolves({ body: text });
return expect(downloadListOfUrls({
url: 'nowhere',
diff --git a/types/crawlers/basic_crawler.d.ts b/types/crawlers/basic_crawler.d.ts
index c8fe99012..234184aed 100644
--- a/types/crawlers/basic_crawler.d.ts
+++ b/types/crawlers/basic_crawler.d.ts
@@ -223,8 +223,6 @@ export type HandleFailedRequestInput = {
* **Example usage:**
*
* ```javascript
- * const rp = require('request-promise-native');
- *
* // Prepare a list of URLs to crawl
* const requestList = new Apify.RequestList({
* sources: [
@@ -240,9 +238,10 @@ export type HandleFailedRequestInput = {
* handleRequestFunction: async ({ request }) => {
* // 'request' contains an instance of the Request class
* // Here we simply fetch the HTML of the page and store it to a dataset
+ * const { body } = await Apify.utils.requestAsBrowser(request);
* await Apify.pushData({
* url: request.url,
- * html: await rp(request.url),
+ * html: body,
* })
* },
* });