Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Remove deprecated code, update docs
Ondra Urban7ac5dea2bcf3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3466705ae..8e9582a82 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -36,8 +36,7 @@ to make crawler development faster and safer. Finally, we will take a good lookat the interface of the whole SDK and update it to improve the developer experience.Bug fixes and scraping features will of course keep landing in versions 1.X as well.-Full list of changes:-+## Full list of changes:- **BREAKING:** Removed `puppeteer` from dependencies. If you want to use Puppeteer,you must install it yourself.- **BREAKING:** Removed `PuppeteerPool`. Use [`browser-pool`](https://github.com/apify/browser-pool).@@ -49,11 +48,24 @@ Full list of changes:- **BREAKING:** The `useSessionPool` and `persistCookiesPerSession` options of crawlersare now `true` by default. Explicitly set them to `false` to override the behavior.- **BREAKING:** `Apify.launchPuppeteer()` no longer accepts `LaunchPuppeteerOptions`.- It now accepts `LaunchContext`.+ It now accepts `PuppeteerLaunchContext`.+### New deprecations:- **DEPRECATED:** `PuppeteerCrawlerOptions.gotoFunction`.Use `PuppeteerCrawlerOptions.preNavigationHooks` and `postNavigationHooks`.+### Removals of earlier deprecated functions:+- **BREAKING:** Removed `Apify.utils.puppeteer.enqueueLinks()`. Deprecated in 01/2019.+ Use `Apify.utils.enqueueLinks()`.+- **BREAKING:** Removed `autoscaledPool.(set|get)MaxConcurrency()`. Deprecated in 2019.+ Use `autoscaledPool.maxConcurrency`.+- **BREAKING:** Removed `CheerioCrawlerOptions.requestOptions`. Deprecated in 03/2020.+ Use `CheerioCrawlerOptions.prepareRequestFunction`.+- **BREAKING:** Removed `Launch.requestOptions`. Deprecated in 03/2020.+ Use `CheerioCrawlerOptions.prepareRequestFunction`.+++### New features:- Added `Apify.PlaywrightCrawler` which is almost identical to `PuppeteerCrawler`,but it crawls with the `playwright` library.- Added `Apify.launchPlaywright(launchContext)` helper function.diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex ab77a6d5d..f6865c471 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -182,22 +182,6 @@ class AutoscaledPool {this.systemStatus = new SystemStatus(ssoCopy);}- /**- * @ignore- */- setMaxConcurrency(maxConcurrency) {- this.log.deprecated('setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');- this._maxConcurrency = maxConcurrency;- }-- /**- * @ignore- */- setMinConcurrency(minConcurrency) {- this.log.deprecated('setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');- this._minConcurrency = minConcurrency;- }-/*** Gets the minimum number of tasks running in parallel.*diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex de2cf6070..acec9e794 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -356,9 +356,6 @@ class CheerioCrawler extends BasicCrawler {prepareRequestFunction: ow.optional.function,postResponseFunction: ow.optional.function,persistCookiesPerSession: ow.optional.boolean,-- // Deprecated- requestOptions: ow.optional.object,}/**@@ -381,9 +378,6 @@ class CheerioCrawler extends BasicCrawler {postResponseFunction,persistCookiesPerSession = false,- // Deprecated- requestOptions,-// BasicCrawlerautoscaledPoolOptions = CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS,...basicCrawlerOptions@@ -406,12 +400,6 @@ class CheerioCrawler extends BasicCrawler {this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);- if (requestOptions) {- // DEPRECATED 2020-03-22- this.requestOptions = requestOptions;- this.log.deprecated('options.requestOptions is deprecated. Use options.prepareRequestFunction instead.');- }-if (suggestResponseEncoding && forceResponseEncoding) {this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');}diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.jsindex d42ac649a..acebde4d1 100644--- a/src/crawlers/playwright_crawler.js+++ b/src/crawlers/playwright_crawler.js@@ -5,7 +5,7 @@ import { gotoExtended } from '../playwright_utils';import {apifyOptionsToLaunchOptions,getPlaywrightLauncherOrThrow,- LaunchPlaywrightOptions, // eslint-disable-line+ PlaywrightLaunchContext, // eslint-disable-line} from '../playwright';/**@@ -95,7 +95,7 @@ import {* };* ]* ```- * @property {LaunchPlaywrightOptions} [launchContext]+ * @property {PlaywrightLaunchContext} [launchContext]* The same options as used by {@link Apify#launchPlaywright}.* @property {number} [handlePageTimeoutSecs=60]* Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.@@ -246,6 +246,11 @@ class PlaywrightCrawler extends BrowserCrawler {...browserCrawlerOptions} = options;+ if (launchContext.proxyUrl) {+ throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.'+ + 'Use PlaywrightCrawlerOptions.proxyConfiguration');+ }+browserPoolOptions.browserPlugins = [new PlaywrightPlugin(// eslint-disable-next-linediff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex cfa88b00c..c8560a8c0 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -247,20 +247,18 @@ class PuppeteerCrawler extends BrowserCrawler {const {stealth,stealthOptions,- proxyUrl,launcher,} = launchContext;- if (proxyUrl && proxyConfiguration) {- throw new Error('It is not possible to combine "options.proxyConfiguration" together with '- + 'custom "proxyUrl" option from "options.launchContext".');+ if (launchContext.proxyUrl) {+ throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'+ + 'Use PuppeteerCrawlerOptions.proxyConfiguration');}browserPoolOptions.browserPlugins = [new PuppeteerPlugin(getPuppeteerOrThrow(launcher),{- proxyUrl,launchOptions: apifyOptionsToLaunchOptions(launchContext),},),@@ -280,10 +278,6 @@ class PuppeteerCrawler extends BrowserCrawler {this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');}- if (proxyUrl) {- this.log.deprecated('options.launchContext.proxyUrl is deprecated use the options.proxyConfiguration instead');- }-if (stealth) {this.browserPool.postLaunchHooks.push(async (pageId, browserController) => {// @TODO: We can do this better now. It is not necessary to override the page.diff --git a/src/playwright.js b/src/playwright.jsindex 1181f6344..041277b7c 100644--- a/src/playwright.js+++ b/src/playwright.js@@ -58,16 +58,15 @@ export function apifyOptionsToLaunchOptions(launchContext) {return launchOptions;}-// TODO yin: `@property ...` didn't work. Extend Puppeteer's `LaunchOptions` didn't work. There is a GitHub issue for that:-// https://github.com/Microsoft/TypeScript/issues/20077/**- * Apify extends the launch options of Puppeteer.- * You can use any of the Puppeteer compatible- * [`LaunchOptions`](https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions)- * options in the {@link Apify#launchPuppeteer}- * function and in addition, all the options available below.+ * Apify extends the launch options of Playwright.+ * You can use any of the Playwright compatible+ * [`LaunchOptions`](https://playwright.dev/docs/api/class-browsertype#browsertypelaunchoptions)+ * options by providing the `launchOptions` property.*- * @typedef LaunchPlaywrightOptions+ * @typedef PlaywrightLaunchContext+ * @property {object} [launchOptions]+ * `browserType.launch` [options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)* @property {string} [proxyUrl]* URL to a HTTP proxy server. It must define the port number,* and it may also contain proxy username and password.@@ -83,9 +82,6 @@ export function apifyOptionsToLaunchOptions(launchContext) {* @property {(string|Object)} [launcher]* By default this function uses require("playwright").chromium`.* If you want to use a different browser you can pass it by this property as `require("playwright").firefox- *- * @property {object} [launchOptions]- * `browserType.launch` [options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)*//**@@ -116,10 +112,10 @@ export function apifyOptionsToLaunchOptions(launchContext) {* for details.**- * @param {LaunchPlaywrightOptions} [options]+ * @param {PlaywrightLaunchContext} [options]* Optional settings passed to `browserType.launch()`. In addition to* [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)- * the object may contain our own {@link LaunchPlaywrightOptions} that enable additional features.+ * the object may contain our own {@link PlaywrightLaunchContext} that enable additional features.* @returns {Promise<Browser>}* Promise that resolves to Playwright's `Browser` instance.* @memberof module:Apifydiff --git a/src/puppeteer.js b/src/puppeteer.jsindex 6c39e46d0..6e94810ab 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -85,16 +85,15 @@ export function apifyOptionsToLaunchOptions(launchContext) {return launchOptions;}-// TODO yin: `@property ...` didn't work. Extend Puppeteer's `LaunchOptions` didn't work. There is a GitHub issue for that:-// https://github.com/Microsoft/TypeScript/issues/20077/*** Apify extends the launch options of Puppeteer.* You can use any of the Puppeteer compatible* [`LaunchOptions`](https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions)- * options in the {@link Apify#launchPuppeteer}- * function and in addition, all the options available below.+ * options by providing the `launchOptions` property.*- * @typedef LaunchPuppeteerOptions+ * @typedef PuppeteerLaunchContext+ * @property {object} [launchOptions]+ * `puppeteer.launch` [options](https://pptr.dev/#?product=Puppeteer&version=v5.5.0&show=api-puppeteerlaunchoptions)* @property {string} [proxyUrl]* URL to a HTTP proxy server. It must define the port number,* and it may also contain proxy username and password.@@ -124,8 +123,6 @@ export function apifyOptionsToLaunchOptions(launchContext) {* @property {StealthOptions} [stealthOptions]* Using this configuration, you can disable some of the hiding tricks.* For these settings to take effect `stealth` must be set to true- * @property {object} [launchOptions]- * `puppeteer.launch` [options](https://pptr.dev/#?product=Puppeteer&version=v5.5.0&show=api-puppeteerlaunchoptions)*//**@@ -167,10 +164,10 @@ export function apifyOptionsToLaunchOptions(launchContext) {* For an example of usage, see the [Synchronous run Example](../examples/synchronous-run)* or the [Puppeteer proxy Example](../examples/puppeteer-with-proxy)*- * @param {LaunchPuppeteerOptions} [options]+ * @param {PuppeteerLaunchContext} [options]* Optional settings passed to `puppeteer.launch()`. In addition to* [Puppeteer's options](https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions)- * the object may contain our own {@link LaunchPuppeteerOptions} that enable additional features.+ * the object may contain our own {@link PuppeteerLaunchContext} that enable additional features.* @returns {Promise<Browser>}* Promise that resolves to Puppeteer's `Browser` instance.* @memberof module:Apifydiff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 61bc0c4fe..704dd1f2c 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -521,8 +521,6 @@ const saveSnapshot = async (page, options = {}) => {}};-let logEnqueueLinksDeprecationWarning = true;-/*** A namespace that contains various utilities for* [Puppeteer](https://github.com/puppeteer/puppeteer) - the headless Chrome Node API.@@ -547,14 +545,6 @@ export const puppeteerUtils = {injectFile,injectJQuery,injectUnderscore,- enqueueLinks: async (...args) => {- if (logEnqueueLinksDeprecationWarning) {- log.warning('Using enqueueLinks() from the Apify.utils.puppeteer namespace is deprecated. '- + 'Please use the Apify.utils.enqueueLinks().');- logEnqueueLinksDeprecationWarning = false;- return enqueueLinks(...args);- }- },enqueueLinksByClickingElements,blockRequests,blockResources,diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex d9fc41b8a..c5769e92a 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -138,7 +138,7 @@ describe('PuppeteerCrawler', () => {expect(options.timeout).toEqual(timeoutSecs * 1000);});- test('should throw if launchOptions.proxyUrl and proxyConfiguration is suplied', async () => {+ test('should throw if launchOptions.proxyUrl is supplied', async () => {try {const puppeteerCrawler = new Apify.PuppeteerCrawler({ //eslint-disable-linerequestList,@@ -147,12 +147,10 @@ describe('PuppeteerCrawler', () => {launchContext: {proxyUrl: 'http://localhost@1234',},- proxyConfiguration: await Apify.createProxyConfiguration({ proxyUrls: ['http://localhost@1234'] }),handlePageFunction: async () => {},});} catch (e) {- expect(e.message).toEqual('It is not possible to combine "options.proxyConfiguration" together with '- + 'custom "proxyUrl" option from "options.launchContext".');+ expect(e.message).toMatch('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.');}expect.hasAssertions();diff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.jsindex 552064428..c2a45da81 100644--- a/test/puppeteer_utils.test.js+++ b/test/puppeteer_utils.test.js@@ -128,17 +128,6 @@ describe('Apify.utils.puppeteer', () => {}});- // TODO Remove with 1.0.0. This is here as a backwards comp for moving- // the function from utils.puppeteer to utils.- test('enqueueLinks() exists in this namespace', async () => {- expect(Apify.utils.puppeteer.enqueueLinks).toBeInstanceOf(Function);- try {- await Apify.utils.puppeteer.enqueueLinks({ page: {}, $: () => {} });- } catch (err) {- expect(err.message).toBe('Only one of the parameters "options.page" or "options.$" must be provided!');- }- });-describe('blockRequests()', () => {let browser = null;beforeAll(async () => {