Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implement, fix docs throughout
Ondra Urban72cb100f10cc
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 46cb13261..d9fe6fa57 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,8 @@+xxx+===================+- Added `loadedUrl` property to `Request` that contains the `request.url` after redirects.+- Reduced default `handlePageTimeoutSecs` for both `CheerioCrawler` and `PuppeteerCrawler` to `60` seconds.+0.11.7 / 2018/01/30===================- Apify SDK now logs basic system info when `required`.diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex 2e7977ee4..68e8ec3b5 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -12,7 +12,7 @@ import { getApifyProxyUrl } from './actor';const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,- handlePageTimeoutSecs: 300,+ handlePageTimeoutSecs: 60,handleFailedRequestFunction: ({ request }) => {const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');@@ -140,10 +140,10 @@ const DEFAULT_OPTIONS = {* - `strictSSL`: use `options.ignoreSslErrors`* - `proxy`: use `options.useApifyProxy` or `options.proxyUrls`*- * @param {Number} [options.handlePageTimeoutSecs=300]+ * @param {Number} [options.handlePageTimeoutSecs=60]* Timeout in which the function passed as `options.handlePageFunction` needs to finish, given in seconds.* @param {Number} [options.requestTimeoutSecs=30]- * Timeout in which the function passed as `options.requestFunction` needs to finish, given in seconds.+ * Timeout in which the HTTP request to the resource needs to finish, given in seconds.* @param {Boolean} [options.ignoreSslErrors=false]* If set to true, SSL certificate errors will be ignored. This is dependent on using the default* request function. If using a custom `options.requestFunction`, user needs to implement this functionality.@@ -204,7 +204,6 @@ const DEFAULT_OPTIONS = {class CheerioCrawler {constructor(options = {}) {const {- requestFunction,requestOptions,handlePageFunction,requestTimeoutSecs,@@ -229,8 +228,6 @@ class CheerioCrawler {} = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');- checkParamOrThrow(requestFunction, 'options.requestFunction', 'Maybe Function');- if (requestFunction) log.warning('CheerioCrawler: options.requestFunction is deprecated. Use BasicCrawler instead.');checkParamOrThrow(requestOptions, 'options.requestOptions', 'Maybe Object');checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');@@ -243,11 +240,6 @@ class CheerioCrawler {if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');- this.requestFunction = async ({ request, autoscaledPool }) => {- return requestFunction- ? requestFunction({ request, autoscaledPool })- : this._defaultRequestFunction({ request, autoscaledPool });- };this.requestOptions = requestOptions;this.handlePageFunction = handlePageFunction;this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;@@ -301,15 +293,13 @@ class CheerioCrawler {*/async _handleRequestFunction({ request, autoscaledPool }) {const response = await addTimeoutToPromise(- this.requestFunction({ request, autoscaledPool }),+ this._requestFunction({ request }),this.requestTimeoutMillis,'CheerioCrawler: requestFunction timed out.',);- let html;- if (typeof response === 'string') html = response;- else if (typeof response === 'object' && typeof response.body === 'string') html = response.body;- else throw new Error('CheerioCrawler: requestFunction returned neither string, nor an object with a body property of type string.');+ const html = response.body;+ request.loadedUrl = response.request.uri.href;const $ = cheerio.load(html);return addTimeoutToPromise(@@ -320,12 +310,12 @@ class CheerioCrawler {}/**- * Default request function to be used. It performs optimizations+ * Function to make the HTTP request. It performs optimizations* on the request such as only downloading the request body if the- * received content type+ * received content type matches text/html.* @ignore*/- async _defaultRequestFunction({ request }) {+ async _requestFunction({ request }) {return new Promise((resolve, reject) => {// Using the streaming API of Request to be able to// handle the response based on headers receieved.diff --git a/src/pseudo_url.js b/src/pseudo_url.jsindex 2cd848861..2f2c2a37d 100644--- a/src/pseudo_url.js+++ b/src/pseudo_url.js@@ -58,7 +58,8 @@ const parsePurl = (purl) => {* Currently, the only supported directive is `[RegExp]`,* which defines a JavaScript-style regular expression to match against the URL.*- * The matching of PURLs against URLs is always case insensitive.+ * The matching of Pseudo URL string against URLs is always case insensitive.+ * If you need case sensitive matching, use an appropriate `RegExp` in place of a Pseudo URL string.** For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:*@@ -89,7 +90,9 @@ const parsePurl = (purl) => {* ```** @param {String|RegExp} purl- * Pseudo URL string or a RegExp instance.+ * Pseudo URL string or a `RegExp` instance.+ * Using a `RegExp` instance enables more granular control,+ * such as making the matching case sensitive.* @param {Object} requestTemplate* Options for the new {@link Request} instances created for matching URLs* by the [`utils.enqueueLinks()`](utils#utils.enqueueLinks) function.diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 5db176c02..c19567344 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -7,7 +7,7 @@ import { addTimeoutToPromise } from './utils';const DEFAULT_OPTIONS = {gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),- handlePageTimeoutSecs: 300,+ handlePageTimeoutSecs: 60,handleFailedRequestFunction: ({ request }) => {const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');log.error('PuppeteerCrawler: Request failed and reached maximum retries', details);@@ -110,7 +110,7 @@ const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;* @param {RequestQueue} options.requestQueue* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.* Either `requestList` or `requestQueue` option must be provided (or both).- * @param {Number} [options.handlePageTimeoutSecs=300]+ * @param {Number} [options.handlePageTimeoutSecs=60]* Timeout in which the function passed as `options.handlePageFunction` needs to finish, in seconds.* @param {Function} [options.gotoFunction]* Overrides the function that opens the page in Puppeteer. The function should return the result of Puppeteer's@@ -293,6 +293,7 @@ class PuppeteerCrawler {const page = await this.puppeteerPool.newPage();try {const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool });+ request.loadedUrl = page.url();await addTimeoutToPromise(this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response }),this.handlePageTimeoutMillis,diff --git a/src/request.js b/src/request.jsindex 9a2bf4f52..046a802ed 100644--- a/src/request.js+++ b/src/request.js@@ -71,6 +71,13 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep* Request ID* @property {String} url* URL of the web page to crawl.+ * @property {String} loadedUrl+ * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed+ * to be included.+ *+ * When using {@link PuppeteerCrawler}, meta tag and JavaScript redirects may,+ * or may not be included, depending on their nature. This generally means that redirects,+ * which happen immediately will most likely be included, but delayed redirects will not.* @property {String} uniqueKey* A unique key identifying the request.* Two requests with the same `uniqueKey` are considered as pointing to the same URL.@@ -99,6 +106,7 @@ class Request {const {id,url,+ loadedUrl = null,uniqueKey,method = 'GET',payload = null,@@ -114,6 +122,7 @@ class Request {checkParamOrThrow(id, 'id', 'Maybe String');checkParamOrThrow(url, 'url', 'String');+ checkParamOrThrow(loadedUrl, 'url', 'Maybe String');checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');checkParamOrThrow(method, 'method', 'String');checkParamOrThrow(payload, 'payload', 'Maybe Buffer | String');@@ -130,6 +139,7 @@ class Request {this.id = id;this.url = url;+ this.loadedUrl = loadedUrl;// NOTE: If URL is invalid, computeUniqueKey() returns null which was causing weird errorsthis.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment) || url;this.method = method;diff --git a/src/request_list.js b/src/request_list.jsindex 49455235f..12ac232fe 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -91,10 +91,14 @@ export const SOURCES_PERSISTENCE_KEY = 'REQUEST_LIST_SOURCES';* ]* ```* @param {String} [options.persistStateKey]- * Identifies the keys in the default key-value store under which the `RequestList` persists its- * initial sources and current state. State represents a position of the last scraped request in the list.- * If this is set then `RequestList`persists all of its sources and the state in regular intervals+ * Identifies the key in the default key-value store under which the `RequestList` persists its+ * current state. State represents a position of the last scraped request in the list.+ * If this is set then `RequestList`persists the state in regular intervals* to key value store and loads the state from there in case it is restarted due to an error or system reboot.+ * @param {String} [options.persistSourcesKey]+ * Identifies the key in the default key-value store under which the `RequestList` persists its+ * initial sources. If this is set then `RequestList`persists all of its sources+ * to key value store at initialization and loads them from there in case it is restarted due to an error or system reboot.* @param {Object} [options.state]* The state object that the `RequestList` will be initialized from.* It is in the form as returned by `RequestList.getState()`, such as follows:diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 3bd3392b7..c63c62b45 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -13,6 +13,15 @@ import Apify from '../build/index';chai.use(chaiAsPromised);+// Add common props to mocked request responses.+const responseMock = {+ request: {+ uri: {+ href: 'loadedUrl',+ },+ },+};+/* eslint-disable no-underscore-dangle */describe('CheerioCrawler', () => {const comparator = (a, b) => {@@ -73,53 +82,6 @@ describe('CheerioCrawler', () => {});});- it('should work with custom request function', async () => {- const sources = [- { url: 'http://example.com/?q=0' },- { url: 'http://example.com/?q=1' },- { url: 'http://example.com/?q=2' },- { url: 'http://example.com/?q=3' },- { url: 'http://example.com/?q=4' },- { url: 'http://example.com/?q=5' },- { url: 'http://example.com/?q=6' },- ];- const processed = [];- const failed = [];- const requestList = new Apify.RequestList({ sources });- const requestFunction = async ({ request }) => {- await delayPromise(1);- return `<html><head><title>${request.url[request.url.length - 1]}</title></head><body>Body</body></html>`;- };- const handlePageFunction = async ({ $, html, request }) => {- request.userData.title = $('title').text();- request.userData.html = html;- processed.push(request);- };-- const cheerioCrawler = new Apify.CheerioCrawler({- requestList,- minConcurrency: 2,- maxConcurrency: 2,- handlePageFunction,- requestFunction,- handleFailedRequestFunction: ({ request }) => failed.push(request),- });-- await requestList.initialize();- await cheerioCrawler.run();-- expect(processed).to.have.lengthOf(7);- expect(failed).to.have.lengthOf(0);-- processed.sort(comparator);- processed.forEach((request, id) => {- expect(request.url).to.be.eql(sources[id].url);- expect(request.userData.title).to.be.eql(String(id));- expect(request.userData.html).to.be.a('string');- expect(request.userData.html.length).not.to.be.eql(0);- });- });-describe('should timeout', () => {let ll;before(() => {@@ -214,7 +176,6 @@ describe('CheerioCrawler', () => {let requestList;const originalGet = rqst.get;beforeEach(async () => {- log.setLevel(log.LEVELS.OFF);requestList = new Apify.RequestList({sources: [{ url: 'http://example.com/?q=0' },@@ -227,7 +188,6 @@ describe('CheerioCrawler', () => {});afterEach(() => {- log.setLevel(log.LEVELS.ERROR);requestList = null;rqst.get = originalGet;});@@ -265,6 +225,7 @@ describe('CheerioCrawler', () => {'content-type': 'text/html', // to avoid throwing'content-encoding': 'gzip',};+ Object.assign(response, responseMock);const ee = new EventEmitter();@@ -300,6 +261,7 @@ describe('CheerioCrawler', () => {'content-type': 'text/html', // to avoid throwing'content-encoding': 'deflate',};+ Object.assign(response, responseMock);const ee = new EventEmitter();@@ -339,6 +301,7 @@ describe('CheerioCrawler', () => {'content-type': 'text/html', // to avoid throwing'content-encoding': 'compress',};+ Object.assign(response, responseMock);const ee = new EventEmitter();@@ -677,15 +640,16 @@ describe('CheerioCrawler', () => {const crawler = new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},- requestFunction: async ({ request }) => {- const opts = crawler._getRequestOptions(request);- proxies.push(opts.proxy);- // it needs to return something valid- return 'html';- },proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],});+ crawler._requestFunction = async ({ request }) => {+ const opts = crawler._getRequestOptions(request);+ proxies.push(opts.proxy);+ // it needs to return something valid+ return Object.assign({ body: 'html' }, responseMock);+ };+const shuffled = crawler.proxyUrls;await crawler.run();@@ -706,15 +670,16 @@ describe('CheerioCrawler', () => {const crawler = new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},- requestFunction: async ({ request }) => {- const opts = crawler._getRequestOptions(request);- proxies.push(opts.proxy);- // it needs to return something valid- return 'html';- },useApifyProxy,});+ crawler._requestFunction = async ({ request }) => {+ const opts = crawler._getRequestOptions(request);+ proxies.push(opts.proxy);+ // it needs to return something valid+ return Object.assign({ body: 'html' }, responseMock);+ };+await crawler.run();delete process.env[ENV_VARS.PROXY_PASSWORD];@@ -740,17 +705,18 @@ describe('CheerioCrawler', () => {const crawler = new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},- requestFunction: async ({ request }) => {- const opts = crawler._getRequestOptions(request);- proxies.push(opts.proxy);- // it needs to return something valid- return 'html';- },useApifyProxy,apifyProxyGroups,apifyProxySession,});+ crawler._requestFunction = async ({ request }) => {+ const opts = crawler._getRequestOptions(request);+ proxies.push(opts.proxy);+ // it needs to return something valid+ return Object.assign({ body: 'html' }, responseMock);+ };+await crawler.run();delete process.env[ENV_VARS.PROXY_PASSWORD];@@ -775,7 +741,6 @@ describe('CheerioCrawler', () => {new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},- requestFunction: async () => {},proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],useApifyProxy: true,});@@ -790,7 +755,6 @@ describe('CheerioCrawler', () => {new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},- requestFunction: async () => {},proxyUrls: [],});throw new Error('Invalid error.');@@ -839,17 +803,6 @@ describe('CheerioCrawler', () => {const crawler = new Apify.CheerioCrawler({requestList,- requestFunction: async () => {- const err = new Error();- err.code = 'ERR_ASSERTION';- err.name = 'AssertionError [ERR_ASSERTION]';- err.operator = '==';- err.expected = 0;- err.stack = ('xxx/tunnel-agent/index.js/yyyy');- throwNextTick(err);- // will never resolve- await new Promise((r, rj) => {}); // eslint-disable-line no-unused-vars- },requestTimeoutSecs: 1 / 1000,handlePageFunction: () => {handlePageCalled = true;@@ -859,6 +812,18 @@ describe('CheerioCrawler', () => {},});+ crawler._requestFunction = async () => {+ const err = new Error();+ err.code = 'ERR_ASSERTION';+ err.name = 'AssertionError [ERR_ASSERTION]';+ err.operator = '==';+ err.expected = 0;+ err.stack = ('xxx/tunnel-agent/index.js/yyyy');+ throwNextTick(err);+ // will never resolve+ await new Promise(() => {});+ };+await requestList.initialize();await crawler.run();