Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: update typedoc plugin + fix some broken links
Martin Adámekf18ad914e751
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c012b4460..86f1b2e02 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -299,7 +299,7 @@ const crawler = new BasicCrawler({#### How to use `sendRequest()`?-See [the Got Scraping guide](../guides/got_scraping.mdx).+See [the Got Scraping guide](https://crawlee.dev/docs/guides/got-scraping).#### Removed options@@ -424,7 +424,7 @@ Previously, you were able to have a browser pool that would mix Puppeteer and PlOne small feature worth mentioning is the ability to handle requests with browser crawlers outside the browser. To do that, we can use a combination of `Request.skipNavigation` and `context.sendRequest()`.-Take a look at how to achieve this by checking out the [Skipping navigation for certain requests](../examples/skip-navigation) example!+Take a look at how to achieve this by checking out the [Skipping navigation for certain requests](https://crawlee.dev/docs/examples/skip-navigation) example!### Loggingdiff --git a/packages/browser-pool/README.md b/packages/browser-pool/README.mdindex c71153799..b4ba0b0b0 100644--- a/packages/browser-pool/README.md+++ b/packages/browser-pool/README.md@@ -479,7 +479,7 @@ const browserPool = new BrowserPool({| [options.postPageCloseHooks] | <code>Array.<function()></code> | | Post-page-close hooks allow you to do page related clean up. The hooks are called with two arguments: `pageId`: `string` and `browserController`: [BrowserController](#BrowserController) || [options.useFingerprints] | <code>boolean</code> | <code>false</code> | If true the Browser pool will automatically generate and inject fingerprints to browsers. || [options.fingerprintOptions] | <code>FingerprintOptions </code> | | Fingerprints options that allows customizing the fingerprinting behavior. |-| [options.fingerprintOptions.fingerprintGeneratorOptions] | | | See the [Fingerprint generator]("https://github.com/apify/fingerprint-generator#headergeneratoroptions") documentation. |+| [options.fingerprintOptions.fingerprintGeneratorOptions] | | | See the [Fingerprint generator](https://github.com/apify/fingerprint-generator#headergeneratoroptions) documentation. || [options.fingerprintOptions.useFingerprintCache] | <code>boolean</code> | <code>true</code> | Fingerprints are automatically assigned to an instance of a Session or proxy URL. You can disable this behavior by setting this property to `false`. || [options.fingerprintOptions.fingerprintCacheSize] | <code>number</code> | <code>10000</code> | Maximum number of cached browser fingerprints. |* * *@@ -505,7 +505,7 @@ or their page limits have been exceeded.<a name="BrowserPool+newPageInNewBrowser"></a>#### `browserPool.newPageInNewBrowser(options)` ⇒ <code>Promise.<Page></code>-Unlike [newPage](newPage), `newPageInNewBrowser` always launches a new+Unlike [newPage](#BrowserPool+newPage), `newPageInNewBrowser` always launches a newbrowser to open the page in. Use the `launchOptions` option toconfigure the new browser.diff --git a/website/package-lock.json b/website/package-lock.jsonindex 75d3f53af..309a18260 100644--- a/website/package-lock.json+++ b/website/package-lock.json@@ -11,7 +11,7 @@"@giscus/react": "^2.2.0","clsx": "^1.1.1","docusaurus-gtm-plugin": "^0.0.2",- "docusaurus-plugin-typedoc-api": "2.3.0",+ "docusaurus-plugin-typedoc-api": "2.4.0","prop-types": "^15.8.1","raw-loader": "^4.0.2","react": "^17.0.2",@@ -3837,9 +3837,9 @@}},"node_modules/@vscode/codicons": {- "version": "0.0.31",- "resolved": "https://registry.npmjs.org/@vscode/codicons/-/codicons-0.0.31.tgz",- "integrity": "sha512-fldpXy7pHsQAMlU1pnGI23ypQ6xLk5u6SiABMFoAmlj4f2MR0iwg7C19IB1xvAEGG+dkxOfRSrbKF8ry7QqGQA=="+ "version": "0.0.32",+ "resolved": "https://registry.npmjs.org/@vscode/codicons/-/codicons-0.0.32.tgz",+ "integrity": "sha512-3lgSTWhAzzWN/EPURoY4ZDBEA80OPmnaknNujA3qnI4Iu7AONWd9xF3iE4L+4prIe8E3TUnLQ4pxoaFTEEZNwg=="},"node_modules/@webassemblyjs/ast": {"version": "1.11.1",@@ -5974,16 +5974,16 @@"integrity": "sha512-Xx/df0Ppd5SultlzUj9qlQk2lX9mNVfTb41juyBUPZ1Nc/5dNx+uN0VuLyF4JEObkDRrUY1EFo9fEUDo8I6QOQ=="},"node_modules/docusaurus-plugin-typedoc-api": {- "version": "2.3.0",- "resolved": "https://registry.npmjs.org/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-2.3.0.tgz",- "integrity": "sha512-cQy1y0vwvlJ/Z+7aYblfR8FuwRRUGr0NlJ3NzpBiRz/xBcCsIGTJaGbZqv2Y64DQBMle/Mu6uBXjfhBojFZRxA==",+ "version": "2.4.0",+ "resolved": "https://registry.npmjs.org/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-2.4.0.tgz",+ "integrity": "sha512-eXHcS1gehUE/cvSnBYvb7l5wizd5lO82Lcu6eutofnvbWoNYSajJ9xHZVBBonfKyhGBlGlbk/as0fZieRr8H/g==","dependencies": {"@docusaurus/plugin-content-docs": "^2.0.1","@docusaurus/types": "^2.0.1","@docusaurus/utils": "^2.0.1",- "@vscode/codicons": "^0.0.31",+ "@vscode/codicons": "^0.0.32","marked": "^4.0.18",- "typedoc": "^0.23.8"+ "typedoc": "^0.23.10"},"engines": {"node": ">=14.15.0",@@ -17325,9 +17325,9 @@}},"@vscode/codicons": {- "version": "0.0.31",- "resolved": "https://registry.npmjs.org/@vscode/codicons/-/codicons-0.0.31.tgz",- "integrity": "sha512-fldpXy7pHsQAMlU1pnGI23ypQ6xLk5u6SiABMFoAmlj4f2MR0iwg7C19IB1xvAEGG+dkxOfRSrbKF8ry7QqGQA=="+ "version": "0.0.32",+ "resolved": "https://registry.npmjs.org/@vscode/codicons/-/codicons-0.0.32.tgz",+ "integrity": "sha512-3lgSTWhAzzWN/EPURoY4ZDBEA80OPmnaknNujA3qnI4Iu7AONWd9xF3iE4L+4prIe8E3TUnLQ4pxoaFTEEZNwg=="},"@webassemblyjs/ast": {"version": "1.11.1",@@ -18898,16 +18898,16 @@"integrity": "sha512-Xx/df0Ppd5SultlzUj9qlQk2lX9mNVfTb41juyBUPZ1Nc/5dNx+uN0VuLyF4JEObkDRrUY1EFo9fEUDo8I6QOQ=="},"docusaurus-plugin-typedoc-api": {- "version": "2.3.0",- "resolved": "https://registry.npmjs.org/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-2.3.0.tgz",- "integrity": "sha512-cQy1y0vwvlJ/Z+7aYblfR8FuwRRUGr0NlJ3NzpBiRz/xBcCsIGTJaGbZqv2Y64DQBMle/Mu6uBXjfhBojFZRxA==",+ "version": "2.4.0",+ "resolved": "https://registry.npmjs.org/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-2.4.0.tgz",+ "integrity": "sha512-eXHcS1gehUE/cvSnBYvb7l5wizd5lO82Lcu6eutofnvbWoNYSajJ9xHZVBBonfKyhGBlGlbk/as0fZieRr8H/g==","requires": {"@docusaurus/plugin-content-docs": "^2.0.1","@docusaurus/types": "^2.0.1","@docusaurus/utils": "^2.0.1",- "@vscode/codicons": "^0.0.31",+ "@vscode/codicons": "^0.0.32","marked": "^4.0.18",- "typedoc": "^0.23.8"+ "typedoc": "^0.23.10"}},"dom-converter": {diff --git a/website/package.json b/website/package.jsonindex bea367e4a..1062d6030 100644--- a/website/package.json+++ b/website/package.json@@ -34,7 +34,7 @@"@giscus/react": "^2.2.0","clsx": "^1.1.1","docusaurus-gtm-plugin": "^0.0.2",- "docusaurus-plugin-typedoc-api": "2.3.0",+ "docusaurus-plugin-typedoc-api": "2.4.0","prop-types": "^15.8.1","raw-loader": "^4.0.2","react": "^17.0.2",