Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Update docusaurus, changelog and version
Ondra Urband022dec30b9c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c349cd7c4..248692d03 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,4 +1,4 @@-1.0.0 / 2021/01/XX+1.0.0 / 2021/01/25====================After 3.5 years of rapid development and a lot of breaking changes and deprecations,here comes the result - **Apify SDK v1**. There were two goals for this release. **Stability**diff --git a/package.json b/package.jsonindex 742ededba..d35adfe5f 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "1.0.0",+ "version": "1.0.1","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=10.17.0"diff --git a/website/i18n/en.json b/website/i18n/en.jsonindex 834c6aafd..573e7f2a4 100644--- a/website/i18n/en.json+++ b/website/i18n/en.json@@ -344,9 +344,6 @@"version-0.22.4/api/version-0.22.4-basic-crawler": {"title": "BasicCrawler"},- "version-0.22.4/api/version-0.22.4-browser-crawler": {- "title": "BrowserCrawler"- },"version-0.22.4/api/version-0.22.4-cheerio-crawler": {"title": "CheerioCrawler"},@@ -356,9 +353,6 @@"version-0.22.4/api/version-0.22.4-key-value-store": {"title": "KeyValueStore"},- "version-0.22.4/api/version-0.22.4-live-view-server": {- "title": "LiveViewServer"- },"version-0.22.4/api/version-0.22.4-log": {"title": "utils.log"},@@ -664,6 +658,93 @@},"version-0.22.4/typedefs/version-0.22.4-user-func": {"title": "UserFunc"+ },+ "version-1.0.0/api/version-1.0.0-apify": {+ "title": "Apify"+ },+ "version-1.0.0/api/version-1.0.0-basic-crawler": {+ "title": "BasicCrawler"+ },+ "version-1.0.0/api/version-1.0.0-cheerio-crawler": {+ "title": "CheerioCrawler"+ },+ "version-1.0.0/api/version-1.0.0-dataset": {+ "title": "Dataset"+ },+ "version-1.0.0/api/version-1.0.0-playwright": {+ "title": "utils.playwright"+ },+ "version-1.0.0/api/version-1.0.0-playwright-crawler": {+ "title": "PlaywrightCrawler"+ },+ "version-1.0.0/api/version-1.0.0-proxy-configuration": {+ "title": "ProxyConfiguration"+ },+ "version-1.0.0/api/version-1.0.0-puppeteer-crawler": {+ "title": "PuppeteerCrawler"+ },+ "version-1.0.0/api/version-1.0.0-request": {+ "title": "Request"+ },+ "version-1.0.0/api/version-1.0.0-request-list": {+ "title": "RequestList"+ },+ "version-1.0.0/api/version-1.0.0-request-queue": {+ "title": "RequestQueue"+ },+ "version-1.0.0/guides/version-1.0.0-apify-platform": {+ "title": "Apify Platform"+ },+ "version-1.0.0/guides/version-1.0.0-docker-images": {+ "title": "Running in Docker"+ },+ "version-1.0.0/guides/version-1.0.0-motivation": {+ "title": "Motivation"+ },+ "version-1.0.0/guides/version-1.0.0-quick-start": {+ "title": "Quick Start"+ },+ "version-1.0.0/guides/version-1.0.0-request-storage": {+ "title": "Request Storage"+ },+ "version-1.0.0/guides/version-1.0.0-result-storage": {+ "title": "Result Storage"+ },+ "version-1.0.0/typedefs/version-1.0.0-basic-crawler-options": {+ "title": "BasicCrawlerOptions"+ },+ "version-1.0.0/typedefs/version-1.0.0-browser-launch-context": {+ "title": "BrowserLaunchContext"+ },+ "version-1.0.0/typedefs/version-1.0.0-cheerio-crawler-options": {+ "title": "CheerioCrawlerOptions"+ },+ "version-1.0.0/typedefs/version-1.0.0-cheerio-handle-page-inputs": {+ "title": "CheerioHandlePageInputs"+ },+ "version-1.0.0/typedefs/version-1.0.0-handle-failed-request-input": {+ "title": "HandleFailedRequestInput"+ },+ "version-1.0.0/typedefs/version-1.0.0-handle-request-inputs": {+ "title": "HandleRequestInputs"+ },+ "version-1.0.0/typedefs/version-1.0.0-playwright-crawler-options": {+ "title": "PlaywrightCrawlerOptions"+ },+ "version-1.0.0/typedefs/version-1.0.0-playwright-launch-context": {+ "title": "PlaywrightLaunchContext"+ },+ "version-1.0.0/typedefs/version-1.0.0-post-response-inputs": {+ "title": "PostResponseInputs"+ },+ "version-1.0.0/typedefs/version-1.0.0-prepare-request-inputs": {+ "title": "PrepareRequestInputs"+ },+ "version-1.0.0/typedefs/version-1.0.0-puppeteer-crawler-options": {+ "title": "PuppeteerCrawlerOptions"+ },+ "version-1.0.0/typedefs/version-1.0.0-puppeteer-launch-context": {+ "title": "PuppeteerLaunchContext"}},"links": {diff --git a/website/siteConfig.js b/website/siteConfig.jsindex 73465ad16..ac7c8791d 100644--- a/website/siteConfig.js+++ b/website/siteConfig.js@@ -69,9 +69,13 @@ const siteConfig = {},algolia: {- apiKey: process.env.ALGOLIA_API_KEY,+ // The key is search-only and safe to publish.+ apiKey: '64ce2544769e34add0e6402688c86e92',indexName: 'apify_sdk',- algoliaOptions: {}, // Optional, if provided by Algolia+ // Optional, if provided by Algolia+ algoliaOptions: {+ facetFilters: ['version:VERSION'],+ },},/* Custom fonts for website */@@ -118,21 +122,26 @@ const siteConfig = {'https://buttons.github.io/buttons.js','https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js','/js/code-block-buttons.js',- // {- // src: 'https://static.hotjar.com/c/hotjar-1021435.js?sv=6',- // async: true,- // },],- stylesheets: ['/css/code-block-buttons.css'],// On page navigation for the current documentation page.onPageNav: 'separate',// No .html extensions for paths.cleanUrl: true,+ deletedDocs: {+ "1.0.0": [+ "puppeteer-pool",+ "puppeteer-pool-options",+ "launch-puppeteer-function",+ "launch-puppeteer-options",+ "puppeteer-goto",+ "puppeteer-goto-inputs",+ ]+ },// Open Graph and Twitter card images.- ogImage: 'img/apify_logo.png',- twitterImage: 'img/apify_logo.png',+ ogImage: 'img/apify_logo.svg',+ twitterImage: 'img/apify_logo.svg',// You may provide arbitrary config keys to be used as needed by your// template. For example, if you need your repo's URL...diff --git a/website/versioned_docs/version-0.22.4/api/BrowserCrawler.md b/website/versioned_docs/version-0.22.4/api/BrowserCrawler.mddeleted file mode 100644index 08b462a45..000000000--- a/website/versioned_docs/version-0.22.4/api/BrowserCrawler.md+++ /dev/null@@ -1,133 +0,0 @@-----id: version-0.22.4-browser-crawler-title: BrowserCrawler-original_id: browser-crawler------<a name="browsercrawler"></a>--Provides a simple framework for parallel crawling of web pages using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer) and-[Playwright](https://github.com/microsoft/playwright). The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of URLs-enabling recursive crawling of websites.--Since `BrowserCrawler` uses headless or even headfull browsers to download web pages and extract data, it is useful for crawling of websites that-require to execute JavaScript. If the target website doesn't need JavaScript, consider using [`CheerioCrawler`](../api/cheerio-crawler), which-downloads the pages using raw HTTP requests and is about 10x faster. @TODO: more?--The source URLs are represented using [`Request`](../api/request) objects that are fed from [`RequestList`](../api/request-list) or-[`RequestQueue`](../api/request-queue) instances provided by the-[`BrowserCrawlerOptions.requestList`](../typedefs/browser-crawler-options#requestlist) or-[`BrowserCrawlerOptions.requestQueue`](../typedefs/browser-crawler-options#requestqueue) constructor options, respectively.--If both [`BrowserCrawlerOptions.requestList`](../typedefs/browser-crawler-options#requestlist) and-[`BrowserCrawlerOptions.requestQueue`](../typedefs/browser-crawler-options#requestqueue) are used, the instance first processes URLs from the-[`RequestList`](../api/request-list) and automatically enqueues all of them to [`RequestQueue`](../api/request-queue) before it starts their-processing. This ensures that a single URL is not crawled multiple times.--The crawler finishes when there are no more [`Request`](../api/request) objects to crawl.--`BrowserCrawler` opens a new browser page (i.e. tab or window) for each [`Request`](../api/request) object to crawl and then calls the function-provided by user as the [`BrowserCrawlerOptions.handlePageFunction`](../typedefs/browser-crawler-options#handlepagefunction) option.--New pages are only opened when there is enough free CPU and memory available, using the functionality provided by the-[`AutoscaledPool`](../api/autoscaled-pool) class. All [`AutoscaledPool`](../api/autoscaled-pool) configuration options can be passed to the-[`BrowserCrawlerOptions.autoscaledPoolOptions`](../typedefs/browser-crawler-options#autoscaledpooloptions) parameter of the `BrowserCrawler`-constructor. For user convenience, the `minConcurrency` and `maxConcurrency` [`AutoscaledPoolOptions`](../typedefs/autoscaled-pool-options) are-available directly in the `BrowserCrawler` constructor.--Note that the pool of browser instances is internally managed by the `BrowserPool` class.--**Example usage:**--**Todo:**: ```javascript const crawler = new Apify.PuppeteerCrawler({ requestList, handlePageFunction: async ({ page, request }) => { // This function-is called to extract data from a single web page // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called // 'request' is-an instance of Request class with information about the page to load await Apify.pushData({ title: await page.title(), url: request.url, succeeded:-true, }) }, handleFailedRequestFunction: async ({ request }) => { // This function is called when the crawling of a request failed too many times-await Apify.pushData({ url: request.url, succeeded: false, errors: request.errorMessages, }) }, });--await crawler.run();--```-## Properties-### `stats`-**Type**: [`Statistics`](../api/statistics)--Contains statistics about the current run.--* * *--### `requestList`-**Type**: [`RequestList`](../api/request-list)--A reference to the underlying [`RequestList`](../api/request-list) class that manages the crawler's [`Request`](../api/request)s.- Only available if used by the crawler.--* * *--### `requestQueue`-**Type**: [`RequestQueue`](../api/request-queue)--A reference to the underlying [`RequestQueue`](../api/request-queue) class that manages the crawler's [`Request`](../api/request)s.- Only available if used by the crawler.--* * *--### `sessionPool`-**Type**: [`SessionPool`](../api/session-pool)--A reference to the underlying [`SessionPool`](../api/session-pool) class that manages the crawler's [`Session`](../api/session)s.- Only available if used by the crawler.--* * *--### `proxyConfiguration`-**Type**: [`ProxyConfiguration`](../api/proxy-configuration)--A reference to the underlying [`ProxyConfiguration`](../api/proxy-configuration) class that manages the crawler's proxies.- Only available if used by the crawler.--* * *--### `browserPool`-**Type**: `BrowserPool`--A reference to the underlying `BrowserPool` class that manages the crawler's browsers.- For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).--* * *--### `autoscaledPool`-**Type**: [`AutoscaledPool`](../api/autoscaled-pool)--A reference to the underlying [`AutoscaledPool`](../api/autoscaled-pool) class that manages the concurrency of the crawler.- Note that this property is only initialized after calling the [`CheerioCrawler.run()`](../api/cheerio-crawler#run) function.- You can use it to change the concurrency settings on the fly,- to pause the crawler by calling [`AutoscaledPool.pause()`](../api/autoscaled-pool#pause)- or to abort it by calling [`AutoscaledPool.abort()`](../api/autoscaled-pool#abort).---* * *--<a name="browsercrawler"></a>--## `new BrowserCrawler(options)`-**Parameters**:--- **`options`**: [`BrowserCrawlerOptions`](../typedefs/browser-crawler-options) - All `BrowserCrawler` parameters are passed via an options object.---* * *--<a name="_handlenavigationtimeout"></a>--## `browserCrawler._handleNavigationTimeout(crawlingContext, error)`-Marks session bad in case of navigation timeout.--**Parameters**:--- **`crawlingContext`**: `object`-- **`error`**: `Error`---* * *-```diff --git a/website/versioned_docs/version-0.22.4/api/LiveViewServer.md b/website/versioned_docs/version-0.22.4/api/LiveViewServer.mddeleted file mode 100644index 8cdc1b74f..000000000--- a/website/versioned_docs/version-0.22.4/api/LiveViewServer.md+++ /dev/null@@ -1,119 +0,0 @@-----id: version-0.22.4-live-view-server-title: LiveViewServer-original_id: live-view-server------<a name="liveviewserver"></a>--`LiveViewServer` enables serving of browser snapshots via web sockets. It includes its own client that provides a simple frontend to viewing the-captured snapshots. A snapshot consists of three pieces of information, the currently opened URL, the content of the page (HTML) and its screenshot.--```json-{- "pageUrl": "https://www.example.com",- "htmlContent": "<html><body> ....",- "screenshotIndex": 3,- "createdAt": "2019-04-18T11:50:40.060Z"-}-```--`LiveViewServer` is useful when you want to be able to inspect the current browser status on demand. When no client is connected, the webserver-consumes very low resources so it should have a close to zero impact on performance. Only once a client connects the server will start serving-snapshots. Once no longer needed, it can be disabled again in the client to remove any performance impact.--NOTE: Screenshot taking in browser typically takes around 300ms. So having the `LiveViewServer` always serve snapshots will have a significant impact-on performance.--When using [`PuppeteerPool`](/docs/api/puppeteer-pool), the `LiveViewServer` can be easily used just by providing the `useLiveView = true` option to-the [`PuppeteerPool`](/docs/api/puppeteer-pool). It can also be initiated via [`PuppeteerCrawler`](/docs/api/puppeteer-crawler)-`puppeteerPoolOptions`.--It will take snapshots of the first page of the latest browser. Taking snapshots of only a single page improves performance and stability dramatically-in high concurrency situations.--When running locally, it is often best to use a headful browser for debugging, since it provides a better view into the browser, including DevTools,-but `LiveViewServer` works too.-------<a name="liveviewserver"></a>--## `new LiveViewServer([options])`--**Parameters**:--- **`[options]`**: `Object` - All `LiveViewServer` parameters are passed via an options object with the following keys:- - **`[screenshotDirectoryPath]`**: `string` - By default, the screenshots are saved to the `live_view` directory in the Apify local storage- directory. Provide a different absolute path to change the settings.- - **`[maxScreenshotFiles]`**: `number` <code> = 10</code> - Limits the number of screenshots stored by the server. This is to prevent using up- too much disk space.- - **`[snapshotTimeoutSecs]`**: `number` <code> = 3</code> - If a snapshot is not made within the timeout, its creation will be aborted. This is- to prevent pages from being hung up by a stalled screenshot.- - **`[maxSnapshotFrequencySecs]`**: `number` <code> = 2</code> - Use this parameter to further decrease the resource consumption of- `LiveViewServer` by limiting the frequency at which it'll serve snapshots.-------<a name="start"></a>--## `liveViewServer.start()`--Starts the HTTP server with web socket connections enabled. Snapshots will not be created until a client has connected.--**Returns**:--`Promise<void>`-------<a name="stop"></a>--## `liveViewServer.stop()`--Prevents the server from receiving more connections. Existing connections will not be terminated, but the server will not prevent a process exit.--**Returns**:--`Promise<void>`-------<a name="serve"></a>--## `liveViewServer.serve(page)`--Serves a snapshot to all connected clients. Screenshots are not served directly, only their index number which is used by client to retrieve the-screenshot.--Will time out and throw in `options.snapshotTimeoutSecs`.--**Parameters**:--- **`page`**: `Page`--**Returns**:--`Promise<void>`-------<a name="isrunning"></a>--## `liveViewServer.isRunning()`--**Returns**:--`boolean`-------<a name="hasclients"></a>--## `liveViewServer.hasClients()`--**Returns**:--`boolean`-----