Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix and improve proxyConf, update changelog
Ondra Urban063774beaea1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2a2b97a0c..be012ebd1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,10 +6,12 @@ for more information and examples.First large change is a redesigned proxy configuration. `Cheerio` and `Puppeteer` crawlersnow accept a `proxyConfiguration` parameter, which is an instance of `ProxyConfiguration`.-This class now exclusively manages both Apify Proxy and custom proxies.+This class now exclusively manages both Apify Proxy and custom proxies. Visit the new+[proxy management guide](https://sdk.apify.com/docs/guides/proxy-management)We also removed `Apify.utils.getRandomUserAgent()` as it was no longer effective-in avoiding bot detection.+in avoiding bot detection and changed the default values for empty properties in+`Request` instances.- **BREAKING:** Removed `Apify.getApifyProxyUrl()`. To get an Apify Proxy url,use `proxyConfiguration.newUrl([sessionId])`.@@ -18,17 +20,23 @@ in avoiding bot detection.in `requestAsBrowser` and `Apify.launchPuppeteer`.- **BREAKING:** Removed `Apify.utils.getRandomUserAgent()` as it was no longer effectivein avoiding bot detection.+- **BREAKING:** `Request` instances no longer initialize empty properties with `null`,+ which means that:+ - empty `errorMessages` are now represented by `[]`, and+ - empty `loadedUrl`, `payload` and `handledAt` are `undefined`.- Add `Apify.createProxyConfiguration()` `async` function to create `ProxyConfiguration`instances. `ProxyConfiguration` itself is not exposed.- Add `proxyConfiguration` to `CheerioCrawlerOptions`and `PuppeteerCrawlerOptions`.- Add `proxyInfo` to `CheerioHandlePageInputs` and `PuppeteerHandlePageInputs`.You can use this object to retrieve information about the currently used proxy- in `Puppeteer` and `Cheerio` crawlers.+ in `Puppeteer` and `Cheerio` crawlers.+- Add click buttons and scroll up options to `Apify.utils.puppeteer.infiniteScroll()`.- Fixed a bug where intercepted requests would never continue.- Fixed a bug where `Apify.utils.requestAsBrowser()` would get into redirect loops.- Fix `Apify.utils.getMemoryInfo()` crashing the process on AWS Lambda and on systemsrunning in Docker without memory cgroups enabled.+- Update Puppeteer to 3.3.0.0.20.4 / 2020-05-11diff --git a/package.json b/package.jsonindex 731cb8d22..e306eec39 100644--- a/package.json+++ b/package.json@@ -51,7 +51,7 @@"lint:fix": "eslint ./src ./test --ext .js,.jsx --fix"},"dependencies": {- "@apify/http-request": "^2.0.0",+ "@apify/http-request": "^2.0.1","@apify/ps-tree": "^1.1.3","@types/cheerio": "^0.22.18","@types/node": "^12",diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex e08bf3098..ddefc86da 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -1,4 +1,5 @@import { checkParamOrThrow } from 'apify-client/build/utils';+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import * as _ from 'underscore';import PuppeteerPool from '../puppeteer_pool'; // eslint-disable-line import/no-duplicatesimport { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';@@ -20,6 +21,7 @@ import AutoscaledPool, { AutoscaledPoolOptions } from '../autoscaling/autoscaledimport { LaunchPuppeteerOptions } from '../puppeteer'; // eslint-disable-line no-unused-vars,import/namedimport { Session } from '../session_pool/session'; // eslint-disable-line no-unused-varsimport { SessionPoolOptions } from '../session_pool/session_pool';+import { ProxyConfiguration } from '../proxy_configuration';// eslint-enable-line import/no-duplicates/**@@ -250,6 +252,7 @@ class PuppeteerCrawler {checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');+ checkParamPrototypeOrThrow(proxyConfiguration, 'options.proxyConfiguration', ProxyConfiguration, 'ProxyConfiguration', true);if (proxyConfiguration && (launchPuppeteerOptions && launchPuppeteerOptions.proxyUrl)) {throw new Error('It is not possible to combine "options.proxyConfiguration" together with 'diff --git a/src/proxy_configuration.js b/src/proxy_configuration.jsindex 68342b1e2..c53e95d61 100644--- a/src/proxy_configuration.js+++ b/src/proxy_configuration.js@@ -352,14 +352,14 @@ export class ProxyConfiguration {* @ignore*/_callNewUrlFunction(sessionId) {- const urlToReturn = this.newUrlFunction(sessionId);+ let proxyUrl;try {- // eslint-disable-next-line no-new- new URL(urlToReturn);+ proxyUrl = this.newUrlFunction(sessionId);+ new URL(proxyUrl); // eslint-disable-line no-new} catch (err) {- this._throwNewUrlFunctionInvalidReturn(urlToReturn);+ this._throwNewUrlFunctionInvalid(err);}- return urlToReturn;+ return proxyUrl;}/**@@ -410,11 +410,11 @@ export class ProxyConfiguration {/*** Throws invalid custom newUrlFunction return- * @param {string} url+ * @param {Error} err* @ignore*/- _throwNewUrlFunctionInvalidReturn(url) {- throw new Error(`The return value "${url}" of provided "options.newUrlFunction" is not a valid URL.`);+ _throwNewUrlFunctionInvalid(err) {+ throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);}/**diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 6e90ec2cc..be4f08559 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -54,16 +54,20 @@ class PuppeteerInstance { // TODO: this is in progress and it will be refactored}async launch() {- this.browserPromise = new Promise(async (resolve) => {- if (this.sessionPool) {- this.session = await this.sessionPool.getSession();- }+ this.browserPromise = new Promise(async (resolve, reject) => {+ try {+ if (this.sessionPool) {+ this.session = await this.sessionPool.getSession();+ }- if (this.proxyConfiguration) {- this.proxyInfo = this.proxyConfiguration.newProxyInfo(this.session ? this.session.id : undefined);+ if (this.proxyConfiguration) {+ this.proxyInfo = this.proxyConfiguration.newProxyInfo(this.session ? this.session.id : undefined);+ }+ const proxyUrl = this.proxyInfo ? this.proxyInfo.url : null;+ resolve(this.launchPuppeteerFunction({ proxyUrl }));+ } catch (err) {+ reject(err);}- const proxyUrl = this.proxyInfo ? this.proxyInfo.url : null;- resolve(this.launchPuppeteerFunction({ proxyUrl }));});}}diff --git a/src/request_list.js b/src/request_list.jsindex 004c87bd8..3b816a71e 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -368,7 +368,7 @@ export class RequestList {this._addRequest(source);}} catch (err) {- throw new Error(`Loading requests with sourcesFunction failed. Cause:\n${err.stack}`);+ throw new Error(`Loading requests with sourcesFunction failed.\nCause: ${err.message}`);}}}diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.jsindex a6ee0206c..69c4448d9 100644--- a/test/proxy_configuration.test.js+++ b/test/proxy_configuration.test.js@@ -134,7 +134,7 @@ describe('ProxyConfiguration', () => {expect(() => proxyConfiguration.newUrl(123456)).not.toThrowError();});- test('should throw invalid newUrlFunction return value', async () => {+ test('should throw on invalid newUrlFunction', async () => {const newUrlFunction = () => {return 'http://proxy.com:1111*invalid_url';};@@ -146,7 +146,7 @@ describe('ProxyConfiguration', () => {proxyConfiguration.newUrl();throw new Error('wrong error');} catch (err) {- expect(err.message).toMatch('The return value "http://proxy.com:1111*invalid_url"');+ expect(err.message).toMatch('The provided newUrlFunction did not return');}});