Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improve docs
Ondra Urbanc59d3d64adf1
diff --git a/src/actor.js b/src/actor.jsindex 7f7ebca16..f91bb21cc 100644--- a/src/actor.js+++ b/src/actor.js@@ -108,22 +108,18 @@ export const getEnv = () => {** The `Apify.main()` function performs the following actions:*- * <ol>- * <li>When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set),+ * - When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set),* it sets up a connection to listen for platform events.* For example, to get a notification about an imminent migration to another server.* See {@link Apify.events} for details.- * </li>- * <li>It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable+ * - It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable* is defined. If not, the functions sets <code>APIFY_LOCAL_STORAGE_DIR</code> to <code>./apify_storage</code>* inside the current working directory. This is to simplify running code examples.- * </li>- * <li>It invokes the user function passed as the <code>userFunc</code> parameter.</li>- * <li>If the user function returned a promise, waits for it to resolve.</li>- * <li>If the user function throws an exception or some other error is encountered,- * prints error details to console so that they are stored to the log.</li>- * <li>Exits the Node.js process, with zero exit code on success and non-zero on errors.</li>- * </ol>+ * - It invokes the user function passed as the <code>userFunc</code> parameter.+ * - If the user function returned a promise, waits for it to resolve.+ * - If the user function throws an exception or some other error is encountered,+ * prints error details to console so that they are stored to the log.+ * - Exits the Node.js process, with zero exit code on success and non-zero on errors.** The user function can be synchronous:*diff --git a/src/browser_launchers/browser_launcher.js b/src/browser_launchers/browser_launcher.jsindex 47be72753..bdd5ae766 100644--- a/src/browser_launchers/browser_launcher.js+++ b/src/browser_launchers/browser_launcher.js@@ -25,6 +25,7 @@ import { getTypicalChromeExecutablePath, isAtHome } from '../utils';/*** Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.+ * @ignore*/export default class BrowserLauncher {static optionsShape = {diff --git a/src/browser_launchers/playwright_launcher.js b/src/browser_launchers/playwright_launcher.jsindex 39f10cf12..46d95f9c2 100644--- a/src/browser_launchers/playwright_launcher.js+++ b/src/browser_launchers/playwright_launcher.js@@ -8,6 +8,21 @@ import BrowserLauncher from './browser_launcher';* [`LaunchOptions`](https://playwright.dev/docs/api/class-browsertype#browsertypelaunchoptions)* options by providing the `launchOptions` property.*+ * **Example:**+ * ```js+ * // launch a headless Chrome (not Chromium)+ * const launchContext = {+ * // Apify helpers+ * useChrome: true,+ * proxyUrl: 'http://user:[email protected]'+ * // Native Playwright options+ * launchOptions: {+ * headless: true,+ * args: ['--some-flag'],+ * }+ * }+ * ```+ ** @typedef PlaywrightLaunchContext* @property {object} [launchOptions]* `browserType.launch` [options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)@@ -24,12 +39,13 @@ import BrowserLauncher from './browser_launcher';* or defaults to the typical Google Chrome executable location specific for the operating system.* By default, this option is `false`.* @property {Object} [launcher]- * By default this function uses require("playwright").chromium`.- * If you want to use a different browser you can pass it by this property as `require("playwright").firefox+ * By default this function uses `require("playwright").chromium`.+ * If you want to use a different browser you can pass it by this property as e.g. `require("playwright").firefox`*//*** `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.+ * @ignore*/export class PlaywrightLauncher extends BrowserLauncher {static optionsShape = {@@ -68,7 +84,7 @@ export class PlaywrightLauncher extends BrowserLauncher {* - Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,* unless it was already defined by the caller or `APIFY_XVFB` environment variable is set to `1`.* Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running actors.- * - Takes the `proxyUrl` option, validates it and adds it to `args` as `--proxy-server=XXX`.+ * - Takes the `proxyUrl` option, validates it and adds it to `launchOptions` in a proper format.* The proxy URL must define a port number and have one of the following schemes: `http://`,* `https://`, `socks4://` or `socks5://`.* If the proxy is HTTP (i.e. has the `http://` scheme) and contains username or password,@@ -79,8 +95,8 @@ export class PlaywrightLauncher extends BrowserLauncher {** To use this function, you need to have the [Playwright](https://www.npmjs.com/package/playwright)* NPM package installed in your project.- * When running on the Apify cloud, you can achieve that simply- * by using the `apify/actor-node-chrome` base Docker image for your actor - see @TODO:+ * When running on the Apify Platform, you can achieve that simply+ * by using the `apify/actor-node-playwright-*` base Docker image for your actor - see* [Apify Actor documentation](https://docs.apify.com/actor/build#base-images)* for details.*diff --git a/src/browser_launchers/puppeteer_launcher.js b/src/browser_launchers/puppeteer_launcher.jsindex 2138fbb9f..76450b420 100644--- a/src/browser_launchers/puppeteer_launcher.js+++ b/src/browser_launchers/puppeteer_launcher.js@@ -16,6 +16,21 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {* [`LaunchOptions`](https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions)* options by providing the `launchOptions` property.*+ * **Example:**+ * ```js+ * // launch a headless Chrome (not Chromium)+ * const launchContext = {+ * // Apify helpers+ * useChrome: true,+ * proxyUrl: 'http://user:[email protected]'+ * // Native Puppeteer options+ * launchOptions: {+ * headless: true,+ * args: ['--some-flag'],+ * }+ * }+ * ```+ ** @typedef PuppeteerLaunchContext* @property {object} [launchOptions]* `puppeteer.launch` [options](https://pptr.dev/#?product=Puppeteer&version=v5.5.0&show=api-puppeteerlaunchoptions)@@ -51,6 +66,7 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {/*** `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.+ * @ignore*/export class PuppeteerLauncher extends BrowserLauncher {static optionsShape = {@@ -142,15 +158,6 @@ export class PuppeteerLauncher extends BrowserLauncher {* to make the proxy work with headless Chrome. For more information, read the* <a href="https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212"* target="_blank">blog post about proxy-chain library</a>.- * - If `options.useApifyProxy` is `true` then the function generates a URL of- * [Apify Proxy](https://docs.apify.com/proxy)- * based on `options.apifyProxyGroups` and `options.apifyProxySession` and passes it as `options.proxyUrl`.- * - The function adds `--no-sandbox` to `args` to enable running- * headless Chrome in a Docker container on the Apify platform.- * - Sets `defaultViewport` Puppeteer option (if not already set)- * to a more reasonable default for screenshots and debugging.- * You can set `options.defaultViewport` to `null` if you prefer to let Puppeteer- * choose the default viewport size.** To use this function, you need to have the [puppeteer](https://www.npmjs.com/package/puppeteer)* NPM package installed in your project.diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 1ed54d8c8..c7508ba66 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -104,8 +104,8 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;* If you're not sure, just keep the default value and the concurrency will scale up automatically.* @property {number} [maxConcurrency=1000]* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.- * @property {boolean} [useSessionPool=false]- * If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.+ * @property {boolean} [useSessionPool=true]+ * Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.* The session instance will be than available in the `handleRequestFunction`.* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.*/@@ -225,7 +225,7 @@ class BasicCrawler {maxRequestsPerCrawl,autoscaledPoolOptions = {},sessionPoolOptions = {},- useSessionPool = false,+ useSessionPool = true,// AutoscaledPool shorthandsminConcurrency,diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.jsindex 4c743944e..a97d2f659 100644--- a/src/crawlers/browser_crawler.js+++ b/src/crawlers/browser_crawler.js@@ -78,7 +78,7 @@ import {* @property {BrowserPoolOptions} [browserPoolOptions]* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.* You can tweak those to fine-tune browser management.- * @property {boolean} [persistCookiesPerSession=false]+ * @property {boolean} [persistCookiesPerSession=true]* Automatically saves cookies to Session. Works only if Session Pool is used.* @property {ProxyConfiguration} [proxyConfiguration]* If set, `PuppeteerCrawler` will be configured for all connections to use@@ -154,8 +154,8 @@ import {* If you're not sure, just keep the default value and the concurrency will scale up automatically.* @property {number} [maxConcurrency=1000]* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.- * @property {boolean} [useSessionPool=false]- * If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.+ * @property {boolean} [useSessionPool=true]+ * Browser crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.* The session instance will be than available in the `handleRequestFunction`.* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.* @ignore@@ -262,7 +262,6 @@ class BrowserCrawler extends BasicCrawler {} = options;if (!useSessionPool && persistCookiesPerSession) {- // @TODO: Maybe we could also automatically set persistCookiesPerSession to false when useSessionPool is false and log warningthrow new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');}diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.jsindex 14a033cc4..180bd6968 100644--- a/src/crawlers/playwright_crawler.js+++ b/src/crawlers/playwright_crawler.js@@ -91,7 +91,7 @@ import { gotoExtended } from '../playwright_utils';* @property {BrowserPoolOptions} [browserPoolOptions]* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.* You can tweak those to fine-tune browser management.- * @property {boolean} [persistCookiesPerSession=false]+ * @property {boolean} [persistCookiesPerSession=true]* Automatically saves cookies to Session. Works only if Session Pool is used.* @property {ProxyConfiguration} [proxyConfiguration]* If set, `PlaywrightCrawler` will be configured for all connections to use@@ -123,8 +123,8 @@ import { gotoExtended } from '../playwright_utils';* If you're not sure, just keep the default value and the concurrency will scale up automatically.* @property {number} [maxConcurrency=1000]* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.- * @property {boolean} [useSessionPool=false]- * If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.+ * @property {boolean} [useSessionPool=true]+ * Playwright crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.* The session instance will be than available in the `handleRequestFunction`.* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.*/@@ -223,6 +223,10 @@ class PlaywrightCrawler extends BrowserCrawler {launchContext: ow.optional.object,}+ /**+ * @param {PlaywrightCrawlerOptions} options+ * All `PlaywrightCrawler` parameters are passed via an options object.+ */constructor(options = {}) {ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex b2359b183..71b6357da 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -70,7 +70,7 @@ import applyStealthToBrowser from '../stealth/stealth';* @property {BrowserPoolOptions} [browserPoolOptions]* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.* You can tweak those to fine-tune browser management.- * @property {boolean} [persistCookiesPerSession=false]+ * @property {boolean} [persistCookiesPerSession=true]* Automatically saves cookies to Session. Works only if Session Pool is used.* @property {ProxyConfiguration} [proxyConfiguration]* If set, `PuppeteerCrawler` will be configured for all connections to use@@ -128,8 +128,8 @@ import applyStealthToBrowser from '../stealth/stealth';* If you're not sure, just keep the default value and the concurrency will scale up automatically.* @property {number} [maxConcurrency=1000]* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.- * @property {boolean} [useSessionPool=false]- * If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.+ * @property {boolean} [useSessionPool=true]+ * Puppeteer crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.* The session instance will be than available in the `handleRequestFunction`.* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.*/diff --git a/src/playwright_utils.js b/src/playwright_utils.jsindex 6f2083477..ae56deba2 100644--- a/src/playwright_utils.js+++ b/src/playwright_utils.js@@ -59,6 +59,26 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {return page.goto(url, gotoOptions);};+/**+ * A namespace that contains various utilities for+ * [Playwright](https://github.com/microsoft/playwright) - the headless Chrome Node API.+ *+ * **Example usage:**+ *+ * ```javascript+ * const Apify = require('apify');+ * const { playwright } = Apify.utils;+ *+ * // Navigate to https://www.example.com in Playwright with a POST request+ * const browser = await Apify.launchPlaywright();+ * const page = await browser.newPage();+ * await playwright.gotoExtended(page, {+ * url: 'https://example.com,+ * method: 'POST',+ * });+ * ```+ * @namespace playwright+ */export const playwrightUtils = {gotoExtended,};diff --git a/website/i18n/en.json b/website/i18n/en.jsonindex 3e9a6f150..7d52a1855 100644--- a/website/i18n/en.json+++ b/website/i18n/en.json@@ -29,6 +29,9 @@"api/log": {"title": "utils.log"},+ "api/playwright": {+ "title": "utils.playwright"+ },"api/playwright-crawler": {"title": "PlaywrightCrawler"},@@ -188,6 +191,9 @@"typedefs/basic-crawler-options": {"title": "BasicCrawlerOptions"},+ "typedefs/browser-launch-context": {+ "title": "BrowserLaunchContext"+ },"typedefs/cheerio-crawler-options": {"title": "CheerioCrawlerOptions"},diff --git a/website/sidebars.json b/website/sidebars.jsonindex 1dc42b57b..c07be5479 100644--- a/website/sidebars.json+++ b/website/sidebars.json@@ -82,6 +82,7 @@"ids": ["api/utils","api/log",+ "api/playwright","api/puppeteer","api/social"]diff --git a/website/tools/build_docs.js b/website/tools/build_docs.jsindex 2319cac14..c1a58a737 100644--- a/website/tools/build_docs.js+++ b/website/tools/build_docs.js@@ -23,7 +23,7 @@ const toId = (name) => {};const getHeader = (title) => {- const prefix = /puppeteer|social|log/.test(title) ? 'utils.' : '';+ const prefix = /playwright|puppeteer|social|log/.test(title) ? 'utils.' : '';const id = toId(title);return `---\nid: ${id}\ntitle: ${prefix}${title}\n---\n`;};