Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Improve docs, update Changelog

Ondra Urban4db979b3fd9a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd5ee9132..87c74a060 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,14 @@
+0.14.0 / 2019-04-15
+===================
+- **BREAKING CHANGE:** Live View is no longer available by passing `liveView = true` to `launchPuppeteerOptions`.
+- New version of Live View is available by passing the `useLiveView = true` option to `PuppeteerPool`.
+ - Only shows snapshots of a single page from a single browser.
+ - Only makes snapshots when a client is connected, having very low performance impact otherwise.
+- Added `Apify.utils.requestExtended` which extends the popular `request` package with fixes and improvements.
+- Added `Apify.utils.requestLikeBrowser` which uses `requestExtended` and disguises as a request made by browser.
+- Added `Apify.utils.puppeteer.addInterceptRequestHandler` and `removeInterceptRequestHandler` which
+ can be used to add multiple request interception handlers to Puppeteer's pages.
+
0.13.7 / 2019-04-04
===================
- Fix a bug where invalid response from `RequestQueue` would occasionally cause crawlers to crash.
diff --git a/src/index.js b/src/index.js
index 9bce6dbbb..4e6a9006d 100644
--- a/src/index.js
+++ b/src/index.js
@@ -22,6 +22,7 @@ import { socialUtils } from './utils_social';
import { enqueueLinks } from './enqueue_links';
import PseudoUrl from './pseudo_url';
import LiveViewServer from './live_view/live_view_server';
+import { requestExtended, requestLikeBrowser } from './utils_request';
/* globals module */
@@ -115,6 +116,8 @@ module.exports = {
social: socialUtils,
log,
enqueueLinks,
+ requestExtended,
+ requestLikeBrowser,
}),
};
diff --git a/src/puppeteer_request_interception.js b/src/puppeteer_request_interception.js
index 4ad549df3..17ce44de9 100644
--- a/src/puppeteer_request_interception.js
+++ b/src/puppeteer_request_interception.js
@@ -104,8 +104,8 @@ const handleRequest = (request, interceptRequestHandlers) => {
*
* @param {Page} page
* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
- * @param {Function} handler Request interception handler. See @TODO for more information.
- * @return {Promise<undefined>}
+ * @param {Function} handler Request interception handler.
+ * @return {Promise}
*/
export const addInterceptRequestHandler = async (page, handler) => {
checkParamOrThrow(page, 'page', 'Object');
@@ -137,8 +137,8 @@ export const addInterceptRequestHandler = async (page, handler) => {
*
* @param {Page} page
* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
- * @param {Function} middleware Request interception handler. See @TODO for more information.
- * @return {Promise<undefined>}
+ * @param {Function} handler Request interception handler.
+ * @return {Promise}
*/
export const removeInterceptRequestHandler = async (page, handler) => {
const handlersArray = pageInterceptRequestHandlersMap
diff --git a/src/utils_request.js b/src/utils_request.js
index 50c1daf03..3ebe4f97f 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -41,7 +41,7 @@ let tunnelAgentExceptionListener;
* - It support not only Gzip compression, but also Brotli and Deflate. To activate this feature,
* simply add `Accept-Encoding: gzip, deflate, br` to `options.headers` (or a combination).
* - Enables abortion of the request based on the response headers, before the data is downloaded.
- * See `options.abort` parameter.
+ * See `options.abortFunction` parameter.
* - SSL connections over proxy do not leak sockets in CLOSE_WAIT state (https://github.com/request/request/issues/2440)
* - Gzip implementation doesn't fail (https://github.com/apifytech/apify-js/issues/266)
* - There is no tunnel-agent AssertionError (https://github.com/request/tunnel-agent/issues/20)
@@ -102,10 +102,9 @@ let tunnelAgentExceptionListener;
* class and it should return `true` if request should be aborted, or `false` otherwise.
* @param [options.throwOnHttpError=false]
* If set to true function throws and error on 4XX and 5XX response codes.
- * @return {{ response, body }}
- * Returns an object with two properties: `response` is the instance of
- * Node's [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage) class,
- * `body` is a `String`, `Buffer` or `Object`, depending on the `encoding` and `json` options.
+ * @return {http.IncomingMessage}
+ * @memberOf utils
+ * @name requestExtended
*/
export const requestExtended = async (options) => {
suppressTunnelAgentAssertError();
@@ -178,6 +177,7 @@ export const requestExtended = async (options) => {
* @param {String} cType.type
* @param {String} cType.encoding
* @returns {Promise<Error>}
+ * @ignore
*/
async function getMoreErrorInfo(response, cType) {
const { type, encoding } = cType;
@@ -220,7 +220,7 @@ async function getMoreErrorInfo(response, cType) {
* @param {http.IncomingMessage} response
* @param {String} [encoding]
* @returns {Promise<String>}
- * @private
+ * @ignore
*/
async function readStreamIntoString(response, encoding) { // eslint-disable-line class-methods-use-this
const stream = decompress(response);
@@ -242,6 +242,7 @@ async function readStreamIntoString(response, encoding) { // eslint-disable-line
* If the stream data is compressed, decompresses it using the Content-Encoding header.
* @param {http.IncomingMessage} response
* @returns {http.IncomingMessage|Stream} - Decompressed response
+ * @ignore
*/
function decompress(response) {
const compression = response.headers['content-encoding'];
@@ -298,9 +299,9 @@ function decompress(response) {
* @param [options.isMobile]
* If `true`, the function uses User-Agent of a mobile browser.
*
- * @return {response}
- * Returns an object with two properties: `response` is the instance of
- * Node's [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage) class
+ * @return {http.IncomingMessage}
+ * @memberOf utils
+ * @name requestLikeBrowser
*/
export const requestLikeBrowser = async (options) => {
const opts = _.defaults({}, options, REQUEST_LIKE_BROWSER_DEFAULT_OPTIONS);