Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: stealth deprecation (#1314)
petrpatek7d20ac2aa01b
diff --git a/docs/guides/avoid_blocking.md b/docs/guides/avoid_blocking.mdnew file mode 100644index 000000000..de57d6ce1--- /dev/null+++ b/docs/guides/avoid_blocking.md@@ -0,0 +1,47 @@+---+id: avoid-blocking+title: Avoid getting blocked+---++A Scraper might get blocked for numerous reasons. Let's narrow it down to two main reasons. The first one is a bad or blocked IP address. This topic is covered in the [proxy management guide](proxy_management.md). The second reason we will explore more is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) or signatures.+Browser fingerprint is a collection of browser attributes and significant features that can show if your browser is a bot or a real user. Moreover, even most browsers have these unique features that allow the website to track the browser even within different IP addresses. This is the main reason why scrapers should change browser fingerprints while doing browser-based scraping. In addition, it should reduce blocking significantly.++Changing browser fingerprints can be a tedious job. Luckily, Apify SDK provides this feature out of the box with zero configuration necessary. Let's take a look at how it is done.++ Changing browser fingerprints is available in `PuppeteerCrawler` and `PlaywrightCrawler`. You have to pass the `useFingerprints` option to the `browserPoolOptions`.++ ```javascript+const crawler = new Apify.PlaywrightCrawler({+ browserPoolOptions: {+ useFingerprints: true,+ },+})++ ```+Now, it is all set. The fingerprints are going to be generated for the default browser and the operating system. The Crawler can have the generation alghoritm customized to reflect particular browser version and many more. Let's take a look at the example bellow:++ ```javascript+const crawler = new Apify.PlaywrightCrawler({+ browserPoolOptions: {+ useFingerprints: true,+ fingerprintOptions: {+ fingerprintGeneratorOptions: {+ browsers: [+ { name: 'firefox', minVersion: 80 },+ { name: 'chrome', minVersion: 87 },+ 'safari',+ ],+ devices: [+ 'desktop',+ ],+ operatingSystems: [+ 'windows',+ ],+ },+ },+ },+})++ ```+ Fingerprint generator has more options available check out the [Fingerprint generator docs](https://github.com/apify/fingerprint-generator#HeaderGeneratorOptions).+\ No newline at end of filediff --git a/src/browser_launchers/puppeteer_launcher.js b/src/browser_launchers/puppeteer_launcher.jsindex f0c4201d6..c703ca216 100644--- a/src/browser_launchers/puppeteer_launcher.js+++ b/src/browser_launchers/puppeteer_launcher.js@@ -3,6 +3,8 @@ import { PuppeteerNode } from 'puppeteer'; // eslint-disable-line no-unused-varsimport { PuppeteerPlugin } from 'browser-pool';import BrowserLauncher from './browser_launcher';import { isAtHome } from '../utils';+import log from '../utils_log';+import { DEFAULT_USER_AGENT } from '../constants';import applyStealthToBrowser, { StealthOptions } from '../stealth/stealth'; // eslint-disable-line no-unused-vars,import/named@@ -61,9 +63,11 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {* With this option selected, all pages will be opened in a new incognito browser context.* This means they will not share cookies nor cache and their resources will not be throttled by one another.* @property {boolean} [stealth]+ * @deprecated* This setting hides most of the known properties that identify headless Chrome and makes it nearly undetectable.* It is recommended to use it together with the `useChrome` set to `true`.* @property {StealthOptions} [stealthOptions]+ * @deprecated* Using this configuration, you can disable some of the hiding tricks.* For these settings to take effect `stealth` must be set to true*/@@ -116,7 +120,11 @@ export class PuppeteerLauncher extends BrowserLauncher {if (this.stealth) {const { hideWebDriver, ...newStealthOptions } = this.stealthOptions;-+ log.deprecated(+ 'Puppeteer "stealth" and "stealthOptions" are deprecated.'+ + ' You should use fingerprints instead.'+ + ' Checkout the fingerprints guide: https://sdk.apify.com/docs/guides/avoid-blocking-fingerprints',+ );applyStealthToBrowser(browser, newStealthOptions);}diff --git a/website/i18n/en.json b/website/i18n/en.jsonindex 055c76cf9..798c9fd15 100644--- a/website/i18n/en.json+++ b/website/i18n/en.json@@ -5,147 +5,12 @@"previous": "Previous","tagline": "The scalable web crawling, scraping and automation library for JavaScript/Node.js.","docs": {- "api/apify": {- "title": "Apify"- },- "api/apify-call-error": {- "title": "ApifyCallError"- },- "api/autoscaled-pool": {- "title": "AutoscaledPool"- },- "api/basic-crawler": {- "title": "BasicCrawler"- },- "api/browser-plugin": {- "title": "BrowserPlugin"- },- "api/cheerio-crawler": {- "title": "CheerioCrawler"- },- "api/configuration": {- "title": "Configuration"- },- "api/dataset": {- "title": "Dataset"- },- "api/key-value-store": {- "title": "KeyValueStore"- },- "api/log": {- "title": "utils.log"- },- "api/playwright": {- "title": "utils.playwright"- },- "api/playwright-crawler": {- "title": "PlaywrightCrawler"- },- "api/proxy-configuration": {- "title": "ProxyConfiguration"- },- "api/pseudo-url": {- "title": "PseudoUrl"- },- "api/puppeteer": {- "title": "utils.puppeteer"- },- "api/puppeteer-crawler": {- "title": "PuppeteerCrawler"- },- "api/request": {- "title": "Request"- },- "api/request-list": {- "title": "RequestList"- },- "api/request-queue": {- "title": "RequestQueue"- },- "api/session": {- "title": "Session"- },- "api/session-pool": {- "title": "SessionPool"- },- "api/snapshotter": {- "title": "Snapshotter"- },- "api/social": {- "title": "utils.social"- },- "api/statistics": {- "title": "Statistics"- },- "api/system-status": {- "title": "SystemStatus"- },- "api/utils": {- "title": "utils"- },- "examples/accept-user-input": {- "title": "Accept user input"- },- "examples/add-data-to-dataset": {- "title": "Add data to dataset"- },- "examples/basic-crawler": {- "title": "Basic crawler"- },- "examples/call-actor": {- "title": "Call actor"- },- "examples/capture-screenshot": {- "title": "Capture a screenshot"- },- "examples/cheerio-crawler": {- "title": "Cheerio crawler"- },- "examples/crawl-all-links": {- "title": "Crawl all links on a website"- },- "examples/crawl-multiple-urls": {- "title": "Crawl multiple URLs"- },- "examples/crawl-relative-links": {- "title": "Crawl a website with relative links"- },- "examples/crawl-single-url": {- "title": "Crawl a single URL"- },- "examples/crawl-sitemap": {- "title": "Crawl a sitemap"- },- "examples/crawl-some-links": {- "title": "Crawl some links on a website"- },- "examples/forms": {- "title": "Forms"- },- "examples/map-and-reduce": {- "title": "Dataset Map and Reduce methods"- },- "examples/playwright-crawler": {- "title": "Playwright crawler"- },- "examples/puppeteer-crawler": {- "title": "Puppeteer crawler"- },- "examples/puppeteer-recursive-crawl": {- "title": "Puppeteer recursive crawl"- },- "examples/puppeteer-with-proxy": {- "title": "Puppeteer with proxy"- },- "examples/synchronous-run": {- "title": "Synchronous run"- },- "examples/use-stealth-mode": {- "title": "Use stealth mode"- },"guides/apify-platform": {"title": "Apify Platform"},+ "guides/avoid-blocking": {+ "title": "Avoid getting blocked"+ },"guides/docker-images": {"title": "Running in Docker"},@@ -188,225 +53,6 @@"readme/support": {"title": "readme/support"},- "typedefs/abort-function": {- "title": "AbortFunction"- },- "typedefs/actor-run": {- "title": "ActorRun"- },- "typedefs/adhoc-webhook": {- "title": "AdhocWebhook"- },- "typedefs/apify-env": {- "title": "ApifyEnv"- },- "typedefs/autoscaled-pool-options": {- "title": "AutoscaledPoolOptions"- },- "typedefs/basic-crawler-options": {- "title": "BasicCrawlerOptions"- },- "typedefs/browser-crawling-context": {- "title": "BrowserCrawlingContext"- },- "typedefs/browser-handle-page-function": {- "title": "BrowserHandlePageFunction"- },- "typedefs/browser-launch-context": {- "title": "BrowserLaunchContext"- },- "typedefs/cheerio-crawler-options": {- "title": "CheerioCrawlerOptions"- },- "typedefs/cheerio-handle-page": {- "title": "CheerioHandlePage"- },- "typedefs/cheerio-handle-page-inputs": {- "title": "CheerioHandlePageInputs"- },- "typedefs/compiled-script-function": {- "title": "CompiledScriptFunction"- },- "typedefs/compiled-script-params": {- "title": "CompiledScriptParams"- },- "typedefs/crawling-context": {- "title": "CrawlingContext"- },- "typedefs/create-session": {- "title": "CreateSession"- },- "typedefs/dataset-consumer": {- "title": "DatasetConsumer"- },- "typedefs/dataset-content": {- "title": "DatasetContent"- },- "typedefs/dataset-mapper": {- "title": "DatasetMapper"- },- "typedefs/dataset-reducer": {- "title": "DatasetReducer"- },- "typedefs/direct-navigation-options": {- "title": "DirectNavigationOptions"- },- "typedefs/event-types": {- "title": "EventTypes"- },- "typedefs/goto-function": {- "title": "GotoFunction"- },- "typedefs/handle-failed-request": {- "title": "HandleFailedRequest"- },- "typedefs/handle-failed-request-input": {- "title": "HandleFailedRequestInput"- },- "typedefs/handle-request": {- "title": "HandleRequest"- },- "typedefs/handle-request-inputs": {- "title": "HandleRequestInputs"- },- "typedefs/hook": {- "title": "Hook"- },- "typedefs/intercept-handler": {- "title": "InterceptHandler"- },- "typedefs/key-consumer": {- "title": "KeyConsumer"- },- "typedefs/key-value-store-value-types": {- "title": "KeyValueStoreValueTypes"- },- "typedefs/logger-options": {- "title": "LoggerOptions"- },- "typedefs/memory-info": {- "title": "MemoryInfo"- },- "typedefs/playwright-crawler-options": {- "title": "PlaywrightCrawlerOptions"- },- "typedefs/playwright-goto-options": {- "title": "PlaywrightGotoOptions"- },- "typedefs/playwright-handle-page-function": {- "title": "PlaywrightHandlePageFunction"- },- "typedefs/playwright-handle-page-function-param": {- "title": "PlaywrightHandlePageFunctionParam"- },- "typedefs/playwright-hook": {- "title": "PlaywrightHook"- },- "typedefs/playwright-launch-context": {- "title": "PlaywrightLaunchContext"- },- "typedefs/post-response": {- "title": "PostResponse"- },- "typedefs/post-response-inputs": {- "title": "PostResponseInputs"- },- "typedefs/prepare-request": {- "title": "PrepareRequest"- },- "typedefs/prepare-request-inputs": {- "title": "PrepareRequestInputs"- },- "typedefs/proxy-configuration-function": {- "title": "ProxyConfigurationFunction"- },- "typedefs/proxy-configuration-options": {- "title": "ProxyConfigurationOptions"- },- "typedefs/proxy-info": {- "title": "ProxyInfo"- },- "typedefs/puppeteer-cookie": {- "title": "PuppeteerCookie"- },- "typedefs/puppeteer-crawler-options": {- "title": "PuppeteerCrawlerOptions"- },- "typedefs/puppeteer-handle-page": {- "title": "PuppeteerHandlePage"- },- "typedefs/puppeteer-handle-page-function-param": {- "title": "PuppeteerHandlePageFunctionParam"- },- "typedefs/puppeteer-hook": {- "title": "PuppeteerHook"- },- "typedefs/puppeteer-launch-context": {- "title": "PuppeteerLaunchContext"- },- "typedefs/queue-operation-info": {- "title": "QueueOperationInfo"- },- "typedefs/request-as-browser-options": {- "title": "RequestAsBrowserOptions"- },- "typedefs/request-as-browser-result": {- "title": "RequestAsBrowserResult"- },- "typedefs/request-list-options": {- "title": "RequestListOptions"- },- "typedefs/request-list-sources-function": {- "title": "RequestListSourcesFunction"- },- "typedefs/request-list-state": {- "title": "RequestListState"- },- "typedefs/request-options": {- "title": "RequestOptions"- },- "typedefs/request-queue-info": {- "title": "RequestQueueInfo"- },- "typedefs/request-transform": {- "title": "RequestTransform"- },- "typedefs/session-options": {- "title": "SessionOptions"- },- "typedefs/session-pool-options": {- "title": "SessionPoolOptions"- },- "typedefs/session-state": {- "title": "SessionState"- },- "typedefs/snapshotter-options": {- "title": "SnapshotterOptions"- },- "typedefs/social-handles": {- "title": "SocialHandles"- },- "typedefs/statistic-persisted-state": {- "title": "StatisticPersistedState"- },- "typedefs/statistic-state": {- "title": "StatisticState"- },- "typedefs/stealth-options": {- "title": "StealthOptions"- },- "typedefs/system-info": {- "title": "SystemInfo"- },- "typedefs/system-status-options": {- "title": "SystemStatusOptions"- },- "typedefs/user-func": {- "title": "UserFunc"- },- "typedefs/webhook-run": {- "title": "WebhookRun"- },"version-0.22.4/api/version-0.22.4-apify": {"title": "Apify"},diff --git a/website/sidebars.json b/website/sidebars.jsonindex 8cf5923b2..5ca98ef6a 100644--- a/website/sidebars.json+++ b/website/sidebars.json@@ -12,7 +12,8 @@"guides/session-management","guides/type-script-actor","guides/docker-images",- "guides/migration-to-v1"+ "guides/migration-to-v1",+ "guides/avoid-blocking"],"Examples": ["examples/accept-user-input",