Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Initial commit

Ondra Urban43fba73f8046
diff --git a/docs/api/Apify.md b/docs/api/Apify.md
index 98bf5f060..d34008981 100644
--- a/docs/api/Apify.md
+++ b/docs/api/Apify.md
@@ -512,7 +512,7 @@ Apify.main(() => {
If the user function returns a promise, it is considered asynchronous:
```javascript
-const request = require('request-promise');
+const request = require('request-promise-native');
Apify.main(() => {
// My asynchronous function that returns a promise
@@ -525,7 +525,7 @@ Apify.main(() => {
To simplify your code, you can take advantage of the `async`/`await` keywords:
```javascript
-const request = require('request-promise');
+const request = require('request-promise-native');
Apify.main(async () => {
// My asynchronous function
diff --git a/docs/api/BasicCrawler.md b/docs/api/BasicCrawler.md
index 14cfb889a..70d78a137 100644
--- a/docs/api/BasicCrawler.md
+++ b/docs/api/BasicCrawler.md
@@ -29,7 +29,7 @@ parameter of the `BasicCrawler` constructor. For user convenience, the `minConcu
**Example usage:**
```javascript
-const rp = require('request-promise');
+const rp = require('request-promise-native');
// Prepare a list of URLs to crawl
const requestList = new Apify.RequestList({
diff --git a/docs/api/CheerioCrawler.md b/docs/api/CheerioCrawler.md
index 75f280a7a..5fe53b748 100644
--- a/docs/api/CheerioCrawler.md
+++ b/docs/api/CheerioCrawler.md
@@ -23,9 +23,8 @@ to [`RequestQueue`](requestqueue) before it starts their processing. This ensure
The crawler finishes when there are no more [`Request`](request) objects to crawl.
By default, `CheerioCrawler` downloads HTML using the
-<a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> NPM package.
-You can override this behavior by setting the `requestFunction` option. If you want to keep `request-promise`,
-but use different than default options, use the `requestOptions` parameter.
+<a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package.
+You can use the `requestOptions` parameter to pass additional options to `request`.
New requests are only dispatched when there is enough free CPU and memory available,
using the functionality provided by the [`AutoscaledPool`](autoscaledpool) class.
@@ -102,7 +101,7 @@ await crawler.run();
$: Cheerio, // the Cheerio object with parsed HTML
html: String // the raw HTML of the page
request: Request,
- response: Object // a response object with properties such as the HTTP status code
+ response: Object // a http.IncomingMessage object with properties such as the `statusCode`
}
</code></pre><p> With the <a href="request"><code>Request</code></a> object representing the URL to crawl.
If the function returns a promise, it is awaited.</p>
@@ -119,20 +118,19 @@ await crawler.run();
<td colspan="3"><p>Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
Either <a href="requestlist"><code>RequestList</code></a> or <a href="requestqueue"><code>RequestQueue</code></a> must be provided.</p>
</td></tr><tr>
-<td><code>[options.requestFunction]</code></td><td><code>function</code></td><td></td>
-</tr>
-<tr>
-<td colspan="3"><p>Overrides the default function that performs the HTTP request to get the raw HTML needed for Cheerio.
- See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L264">GitHub</a> for default behavior.</p>
-</td></tr><tr>
<td><code>[options.requestOptions]</code></td><td><code>Object</code></td><td></td>
</tr>
<tr>
-<td colspan="3"><p>Represents the options passed to the <code>requestFunction</code>, which are essentially the options passed to
- <a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> to make the HTTP call.
- Provided <code>requestOptions</code> are merged with defaults so if you only need to add a parameter, there&#39;s no need to duplicate
- the whole object.</p>
-</td></tr><tr>
+<td colspan="3"><p>Represents the options passed to
+ <a href="https://www.npmjs.com/package/request" target="_blank">request</a> to make the HTTP call.
+ Provided <code>requestOptions</code> are added to internal defaults that cannot be overridden to ensure
+ the operation of <code>CheerioCrawler</code> and all its options. If you need more granular control over
+ your requests, use <a href="basiccrawler"><code>BasicCrawler</code></a>.</p>
+<p> The internal defaults include:</p>
+<pre><code> - `url`, `method`, `headers`: provided by `requestList` and/or `requestQueue`
+ - `strictSSL`: use `options.ignoreSslErrors`
+ - `proxy`: use `options.useApifyProxy` or `options.proxyUrls`
+</code></pre></td></tr><tr>
<td><code>[options.handlePageTimeoutSecs]</code></td><td><code>Number</code></td><td><code>300</code></td>
</tr>
<tr>
diff --git a/docs/api/Request.md b/docs/api/Request.md
index 52a0c0647..c0038c977 100644
--- a/docs/api/Request.md
+++ b/docs/api/Request.md
@@ -101,7 +101,7 @@ const foo = request.userData.foo;
* [Request](request)
* [`new Request(options)`](#new_Request_new)
- * [`.pushErrorMessage(errorOrMessage)`](#Request+pushErrorMessage)
+ * [`.pushErrorMessage(errorOrMessage, [options])`](#Request+pushErrorMessage)
<a name="new_Request_new"></a>
@@ -181,7 +181,7 @@ to override the default behavior and specify which URLs shall be considered equa
</table>
<a name="Request+pushErrorMessage"></a>
-## `request.pushErrorMessage(errorOrMessage)`
+## `request.pushErrorMessage(errorOrMessage, [options])`
Stores information about an error that occurred during processing of this request.
You should always use Error instances when throwing errors in JavaScript.
@@ -194,14 +194,23 @@ as possible, since just throwing a bad type error makes any debugging rather dif
<table>
<thead>
<tr>
-<th>Param</th><th>Type</th>
+<th>Param</th><th>Type</th><th>Default</th>
</tr>
</thead>
<tbody>
<tr>
-<td><code>errorOrMessage</code></td><td><code>Error</code> | <code>String</code></td>
+<td><code>errorOrMessage</code></td><td><code>Error</code> | <code>String</code></td><td></td>
</tr>
<tr>
<td colspan="3"><p>Error object or error message to be stored in the request.</p>
+</td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.omitStack]</code></td><td><code>Boolean</code></td><td><code>false</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Only push the error message without stack trace when true.</p>
</td></tr></tbody>
</table>
diff --git a/docs/examples/PuppeteerCrawler.md b/docs/examples/PuppeteerCrawler.md
index ee8b8ec40..db5f184bb 100644
--- a/docs/examples/PuppeteerCrawler.md
+++ b/docs/examples/PuppeteerCrawler.md
@@ -17,7 +17,7 @@ Apify.main(async () => {
// Create and initialize an instance of the RequestList class that contains the start URL.
const requestList = new Apify.RequestList({
sources: [
- { url: 'http://www.ycombinator.com/' },
+ { url: 'https://news.ycombinator.com/' },
],
});
await requestList.initialize();
@@ -33,9 +33,12 @@ Apify.main(async () => {
requestList,
requestQueue,
- // Run Puppeteer in headless mode. If you set headless to false, you'll see the scraping
- // browsers showing up on your screen. This is great for debugging.
- launchPuppeteerOptions: { headless: true },
+ // Here you can set options that are passed to the Apify.launchPuppeteer() function.
+ // For example, you can set "slowMo" to slow down Puppeteer operations to simplify debugging
+ launchPuppeteerOptions: { slowMo: 500 },
+
+ // Stop crawling after several pages
+ maxRequestsPerCrawl: 10,
// This function will be called for each URL to crawl.
// Here you can write the Puppeteer scripts you are familiar with,
diff --git a/package.json b/package.json
index 0c8865b1c..0c861975f 100644
--- a/package.json
+++ b/package.json
@@ -49,7 +49,7 @@
},
"dependencies": {
"@apify/ps-tree": "^1.1.2",
- "apify-client": "^0.3.3",
+ "apify-client": "^0.3.4",
"apify-shared": "^0.1.7",
"bluebird": "^3.5.0",
"cheerio": "^1.0.0-rc.2",
@@ -58,7 +58,8 @@
"jquery": "^3.3.1",
"mime": "^2.3.1",
"proxy-chain": "^0.2.4",
- "request-promise": "^4.2.2",
+ "request": "^2.88.0",
+ "request-promise-native": "^1.0.5",
"rimraf": "^2.6.1",
"underscore": "^1.9.0",
"ws": "^6.0.0",
diff --git a/src/actor.js b/src/actor.js
index 4a3890596..5d8ffe669 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -118,7 +118,7 @@ export const getEnv = () => {
*
* If the user function returns a promise, it is considered asynchronous:
* ```javascript
- * const request = require('request-promise');
+ * const request = require('request-promise-native');
*
* Apify.main(() => {
* // My asynchronous function that returns a promise
@@ -131,7 +131,7 @@ export const getEnv = () => {
* To simplify your code, you can take advantage of the `async`/`await` keywords:
*
* ```javascript
- * const request = require('request-promise');
+ * const request = require('request-promise-native');
*
* Apify.main(async () => {
* // My asynchronous function
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 4ee61d9bc..caca192d8 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -41,7 +41,7 @@ const DEFAULT_OPTIONS = {
* **Example usage:**
*
* ```javascript
- * const rp = require('request-promise');
+ * const rp = require('request-promise-native');
*
* // Prepare a list of URLs to crawl
* const requestList = new Apify.RequestList({
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index 604f8f130..1a9af4775 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -1,6 +1,7 @@
-import rp from 'request-promise';
+import rqst from 'request';
import _ from 'underscore';
import cheerio from 'cheerio';
+import contentType from 'content-type';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import BasicCrawler from './basic_crawler';
@@ -39,9 +40,8 @@ const DEFAULT_OPTIONS = {
* The crawler finishes when there are no more {@link Request} objects to crawl.
*
* By default, `CheerioCrawler` downloads HTML using the
- * <a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> NPM package.
- * You can override this behavior by setting the `requestFunction` option. If you want to keep `request-promise`,
- * but use different than default options, use the `requestOptions` parameter.
+ * <a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package.
+ * You can use the `requestOptions` parameter to pass additional options to `request`.
*
* New requests are only dispatched when there is enough free CPU and memory available,
* using the functionality provided by the {@link AutoscaledPool} class.
@@ -97,7 +97,7 @@ const DEFAULT_OPTIONS = {
* $: Cheerio, // the Cheerio object with parsed HTML
* html: String // the raw HTML of the page
* request: Request,
- * response: Object // a response object with properties such as the HTTP status code
+ * response: Object // a http.IncomingMessage object with properties such as the `statusCode`
* }
* ```
* With the {@link Request} object representing the URL to crawl.
@@ -108,14 +108,18 @@ const DEFAULT_OPTIONS = {
* @param {RequestQueue} options.requestQueue
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
* Either {@link RequestList} or {@link RequestQueue} must be provided.
- * @param {Function} [options.requestFunction]
- * Overrides the default function that performs the HTTP request to get the raw HTML needed for Cheerio.
- * See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L264">GitHub</a> for default behavior.
* @param {Object} [options.requestOptions]
- * Represents the options passed to the `requestFunction`, which are essentially the options passed to
- * <a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> to make the HTTP call.
- * Provided `requestOptions` are merged with defaults so if you only need to add a parameter, there's no need to duplicate
- * the whole object.
+ * Represents the options passed to
+ * <a href="https://www.npmjs.com/package/request" target="_blank">request</a> to make the HTTP call.
+ * Provided `requestOptions` are added to internal defaults that cannot be overridden to ensure
+ * the operation of `CheerioCrawler` and all its options. If you need more granular control over
+ * your requests, use {@link BasicCrawler}.
+ *
+ * The internal defaults include:
+ * - `url`, `method`, `headers`: provided by `requestList` and/or `requestQueue`
+ * - `strictSSL`: use `options.ignoreSslErrors`
+ * - `proxy`: use `options.useApifyProxy` or `options.proxyUrls`
+ *
* @param {Number} [options.handlePageTimeoutSecs=300]
* Timeout in which the function passed as `options.handlePageFunction` needs to finish, given in seconds.
* @param {Number} [options.requestTimeoutSecs=30]
@@ -189,6 +193,7 @@ class CheerioCrawler {
checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');
checkParamOrThrow(requestFunction, 'options.requestFunction', 'Maybe Function');
+ if (requestFunction) log.warning('CheerioCrawler: options.requestFunction is deprecated. Use BasicCrawler instead.');
checkParamOrThrow(requestOptions, 'options.requestOptions', 'Maybe Object');
checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');
checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');
@@ -288,6 +293,7 @@ class CheerioCrawler {
this.rejectOnAbortPromise,
]);
+
let html;
if (typeof response === 'string') html = response;
else if (typeof response === 'object' && typeof response.body === 'string') html = response.body;
@@ -306,7 +312,18 @@ class CheerioCrawler {
* @ignore
*/
async _defaultRequestFunction({ request }) {
- return rp(this._getRequestOptions(request));
+ return new Promise((resolve, reject) => {
+ rqst(this._getRequestOptions(request))
+ .on('response', (res) => {
+ const { type, encoding } = contentType(res);
+
+
+ if (res.statusCode >= 500) {
+
+ }
+ })
+ .on('error', err => reject(err));
+ });
}
/**
@@ -315,7 +332,7 @@ class CheerioCrawler {
* @ignore
*/
_getRequestOptions(request) {
- const defaultRequestOptions = {
+ const mandatoryRequestOptions = {
url: request.url,
method: request.method,
headers: request.headers,
@@ -324,7 +341,7 @@ class CheerioCrawler {
simple: false,
proxy: this._getProxyUrl(),
};
- return _.defaults({}, this.requestOptions, defaultRequestOptions);
+ return Object.assign({}, this.requestOptions, mandatoryRequestOptions);
}
/**
@@ -346,6 +363,19 @@ class CheerioCrawler {
}
return null;
}
+
+ async _readStreamIntoString(stream, encoding) { // eslint-disable-line class-methods-use-this
+ return new Promise((resolve, reject) => {
+ const chunks = [];
+ stream
+ .on('data', chunk => chunks.push(chunk))
+ .on('error', err => reject(err))
+ .on('end', () => {
+ const buffer = Buffer.concat(chunks);
+ resolve(buffer.toString(encoding));
+ });
+ });
+ }
}
export default CheerioCrawler;
diff --git a/src/utils.js b/src/utils.js
index 556c7b414..f8a463898 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -5,7 +5,7 @@ import fs from 'fs';
import fsExtra from 'fs-extra';
import ApifyClient from 'apify-client';
import psTree from '@apify/ps-tree';
-import requestPromise from 'request-promise';
+import requestPromise from 'request-promise-native';
import XRegExp from 'xregexp';
import { delayPromise, getRandomInt } from 'apify-shared/utilities';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';