Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: Add llms.txt and llms-full.txt generator (#3061)
Jan Turoň79f3ba529f03
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.jsindex a054a387a..8d888b151 100644--- a/website/docusaurus.config.js+++ b/website/docusaurus.config.js@@ -172,6 +172,20 @@ module.exports = {id: 'GTM-5P7MCS7',},],+ [+ '@signalwire/docusaurus-plugin-llms-txt',+ {+ enableDescriptions: false,+ content: {+ excludeRoutes: ['/js/api/3.*/**', '/js/api/3.*', '/js/api/next/**', '/js/api/next'],+ includeVersionedDocs: false,+ enableLlmsFullTxt: true,+ includeBlog: true,+ includePages: true,+ relativePaths: false,+ },+ }+ ],async function runnableCodeBlock() {return {name: 'runnable-code-block',diff --git a/website/package.json b/website/package.jsonindex c93d591cb..53417495d 100644--- a/website/package.json+++ b/website/package.json@@ -1,6 +1,7 @@{"scripts": {"examples": "docusaurus-examples",+ "postbuild": "sed -i '' 's/\\.md//g' build/llms.txt build/llms-full.txt","postinstall": "npx patch-package","start": "rimraf .docusaurus && docusaurus start","start:fast": "rimraf .docusaurus && CRAWLEE_DOCS_FAST=1 docusaurus start",@@ -44,6 +45,7 @@"@docusaurus/theme-common": "3.8.1","@giscus/react": "^3.0.0","@mdx-js/react": "^3.0.1",+ "@signalwire/docusaurus-plugin-llms-txt": "^1.2.1","axios": "^1.5.0","buffer": "^6.0.3","clsx": "^2.0.0",diff --git a/website/yarn.lock b/website/yarn.lockindex b4b7ee1ba..77c773022 100644--- a/website/yarn.lock+++ b/website/yarn.lock@@ -3438,6 +3438,28 @@ __metadata:languageName: nodelinkType: hard+"@signalwire/docusaurus-plugin-llms-txt@npm:^1.2.1":+ version: 1.2.1+ resolution: "@signalwire/docusaurus-plugin-llms-txt@npm:1.2.1"+ dependencies:+ fs-extra: "npm:^11.0.0"+ hast-util-select: "npm:^6.0.4"+ hast-util-to-html: "npm:^9.0.5"+ hast-util-to-string: "npm:^3.0.1"+ p-map: "npm:^7.0.2"+ rehype-parse: "npm:^9"+ rehype-remark: "npm:^10"+ remark-gfm: "npm:^4"+ remark-stringify: "npm:^11"+ string-width: "npm:^5.0.0"+ unified: "npm:^11"+ unist-util-visit: "npm:^5"+ peerDependencies:+ "@docusaurus/core": ^3.0.0+ checksum: 10c0/a9847760594f6ae2274c5130225e7dfc3220dd85ffefbf33ba0086a98649732b09409a1607a5b4ba814102413eb0d5a2d2365405f38bb89d7f68e34166acbebd+ languageName: node+ linkType: hard+"@sinclair/typebox@npm:^0.27.8":version: 0.27.8resolution: "@sinclair/typebox@npm:0.27.8"@@ -5350,6 +5372,13 @@ __metadata:languageName: nodelinkType: hard+"bcp-47-match@npm:^2.0.0":+ version: 2.0.3+ resolution: "bcp-47-match@npm:2.0.3"+ checksum: 10c0/ae5c202854df8a9ad4777dc3b49562578495a69164869f365a88c1a089837a9fbbce4c0c44f6f1a5e44c7841f47e91fe6fea00306ca49ce5ec95a7eb71f839c4+ languageName: node+ linkType: hard+"big.js@npm:^5.2.2":version: 5.2.2resolution: "big.js@npm:5.2.2"@@ -6417,6 +6446,13 @@ __metadata:languageName: nodelinkType: hard+"css-selector-parser@npm:^3.0.0":+ version: 3.1.3+ resolution: "css-selector-parser@npm:3.1.3"+ checksum: 10c0/0bba96edfd27827d79933b113c42bec627b96a79f6fe4b12dec12da109d0b3a25f2f76d385b7c28ff22dca68840251751d1061d9226657755430e4787bf4594e+ languageName: node+ linkType: hard+"css-tree@npm:^2.3.1":version: 2.3.1resolution: "css-tree@npm:2.3.1"@@ -6816,6 +6852,15 @@ __metadata:languageName: nodelinkType: hard+"direction@npm:^2.0.0":+ version: 2.0.1+ resolution: "direction@npm:2.0.1"+ bin:+ direction: cli.js+ checksum: 10c0/dce809431cad978e0778769a3818ea797ebe0bd542c85032ad9ad98971e2021a146be62feb259d7ffe4b76739e07b23e861b29c3f184ac8d38cc6ba956d5c586+ languageName: node+ linkType: hard+"dns-packet@npm:^5.2.2":version: 5.6.1resolution: "dns-packet@npm:5.6.1"@@ -8155,7 +8200,7 @@ __metadata:languageName: nodelinkType: hard-"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0":+"fs-extra@npm:^11.0.0, fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0":version: 11.3.0resolution: "fs-extra@npm:11.3.0"dependencies:@@ -8645,6 +8690,30 @@ __metadata:languageName: nodelinkType: hard+"hast-util-embedded@npm:^3.0.0":+ version: 3.0.0+ resolution: "hast-util-embedded@npm:3.0.0"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ hast-util-is-element: "npm:^3.0.0"+ checksum: 10c0/054c3d3b96fcd5c1d1c6f8d38ce1f7f33022ba6362129a022673d0b539f876acdcababbb9df29812fb927294f98ef7a2f44519a80d637fe3eea1819c9e69eeac+ languageName: node+ linkType: hard++"hast-util-from-html@npm:^2.0.0":+ version: 2.0.3+ resolution: "hast-util-from-html@npm:2.0.3"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ devlop: "npm:^1.1.0"+ hast-util-from-parse5: "npm:^8.0.0"+ parse5: "npm:^7.0.0"+ vfile: "npm:^6.0.0"+ vfile-message: "npm:^4.0.0"+ checksum: 10c0/993ef707c1a12474c8d4094fc9706a72826c660a7e308ea54c50ad893353d32e139b7cbc67510c2e82feac572b320e3b05aeb13d0f9c6302d61261f337b46764+ languageName: node+ linkType: hard+"hast-util-from-parse5@npm:^8.0.0":version: 8.0.3resolution: "hast-util-from-parse5@npm:8.0.3"@@ -8661,6 +8730,46 @@ __metadata:languageName: nodelinkType: hard+"hast-util-has-property@npm:^3.0.0":+ version: 3.0.0+ resolution: "hast-util-has-property@npm:3.0.0"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ checksum: 10c0/6e2c0e22ca893c6ebb60f8390e184c4deb041c36d09796756f02cd121c1789c0f5c862ed06caea8f1a80ea8c0ef6a7854dd57946c2eebb76488727bd4a1c952e+ languageName: node+ linkType: hard++"hast-util-is-body-ok-link@npm:^3.0.0":+ version: 3.0.1+ resolution: "hast-util-is-body-ok-link@npm:3.0.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ checksum: 10c0/c320cbd9a9a834b007a6f2f8c271e98b8331c0193adf06e0a7c5ea0acae664e97ce28eb4436e0658bc5cdb8f47390ec1c6cba7c4fe1ded10951fcdd1432f60bf+ languageName: node+ linkType: hard++"hast-util-is-element@npm:^3.0.0":+ version: 3.0.0+ resolution: "hast-util-is-element@npm:3.0.0"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ checksum: 10c0/f5361e4c9859c587ca8eb0d8343492f3077ccaa0f58a44cd09f35d5038f94d65152288dcd0c19336ef2c9491ec4d4e45fde2176b05293437021570aa0bc3613b+ languageName: node+ linkType: hard++"hast-util-minify-whitespace@npm:^1.0.0":+ version: 1.0.1+ resolution: "hast-util-minify-whitespace@npm:1.0.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ hast-util-embedded: "npm:^3.0.0"+ hast-util-is-element: "npm:^3.0.0"+ hast-util-whitespace: "npm:^3.0.0"+ unist-util-is: "npm:^6.0.0"+ checksum: 10c0/20a7d64947e080463084f444ad09c7f28c40e7648ca2d9c6c036e42a67f8e945d352560ff599304c988257c1e477abcf6a1f508c0900211fa58ec1ba21b36533+ languageName: node+ linkType: hard+"hast-util-parse-selector@npm:^4.0.0":version: 4.0.0resolution: "hast-util-parse-selector@npm:4.0.0"@@ -8670,6 +8779,19 @@ __metadata:languageName: nodelinkType: hard+"hast-util-phrasing@npm:^3.0.0":+ version: 3.0.1+ resolution: "hast-util-phrasing@npm:3.0.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ hast-util-embedded: "npm:^3.0.0"+ hast-util-has-property: "npm:^3.0.0"+ hast-util-is-body-ok-link: "npm:^3.0.0"+ hast-util-is-element: "npm:^3.0.0"+ checksum: 10c0/d77e186ea3d7d62f6db9c4a55c3e6d9f1f6affd5f40250e8de9d73f167ae19fcc02fafe1601dfbe36e90f76ed5013ac004f0b6b398aee3a04a7a81de12788600+ languageName: node+ linkType: hard+"hast-util-raw@npm:^9.0.0":version: 9.1.0resolution: "hast-util-raw@npm:9.1.0"@@ -8691,6 +8813,29 @@ __metadata:languageName: nodelinkType: hard+"hast-util-select@npm:^6.0.4":+ version: 6.0.4+ resolution: "hast-util-select@npm:6.0.4"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ "@types/unist": "npm:^3.0.0"+ bcp-47-match: "npm:^2.0.0"+ comma-separated-tokens: "npm:^2.0.0"+ css-selector-parser: "npm:^3.0.0"+ devlop: "npm:^1.0.0"+ direction: "npm:^2.0.0"+ hast-util-has-property: "npm:^3.0.0"+ hast-util-to-string: "npm:^3.0.0"+ hast-util-whitespace: "npm:^3.0.0"+ nth-check: "npm:^2.0.0"+ property-information: "npm:^7.0.0"+ space-separated-tokens: "npm:^2.0.0"+ unist-util-visit: "npm:^5.0.0"+ zwitch: "npm:^2.0.0"+ checksum: 10c0/d6829953f829c24ffe465c2b156f6a7cd352f7d9b4d601e0e6ca38b85cc4a720bb9f027d34881c3b2a05f4b55c9375e256dbf43ca88604230da784e1c9c7d03f+ languageName: node+ linkType: hard+"hast-util-to-estree@npm:^3.0.0":version: 3.1.3resolution: "hast-util-to-estree@npm:3.1.3"@@ -8715,7 +8860,7 @@ __metadata:languageName: nodelinkType: hard-"hast-util-to-html@npm:^9.0.4":+"hast-util-to-html@npm:^9.0.0, hast-util-to-html@npm:^9.0.4, hast-util-to-html@npm:^9.0.5":version: 9.0.5resolution: "hast-util-to-html@npm:9.0.5"dependencies:@@ -8757,6 +8902,28 @@ __metadata:languageName: nodelinkType: hard+"hast-util-to-mdast@npm:^10.0.0":+ version: 10.1.2+ resolution: "hast-util-to-mdast@npm:10.1.2"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ "@types/mdast": "npm:^4.0.0"+ "@ungap/structured-clone": "npm:^1.0.0"+ hast-util-phrasing: "npm:^3.0.0"+ hast-util-to-html: "npm:^9.0.0"+ hast-util-to-text: "npm:^4.0.0"+ hast-util-whitespace: "npm:^3.0.0"+ mdast-util-phrasing: "npm:^4.0.0"+ mdast-util-to-hast: "npm:^13.0.0"+ mdast-util-to-string: "npm:^4.0.0"+ rehype-minify-whitespace: "npm:^6.0.0"+ trim-trailing-lines: "npm:^2.0.0"+ unist-util-position: "npm:^5.0.0"+ unist-util-visit: "npm:^5.0.0"+ checksum: 10c0/2edd4521b147734078d66e03cd43c571a0a3aeefd3fcc34659c783b25e9222ddb5c8c759b12a86ebc70a25b3888505dc59b913ff36ae17cca04d52050592a963+ languageName: node+ linkType: hard+"hast-util-to-parse5@npm:^8.0.0":version: 8.0.0resolution: "hast-util-to-parse5@npm:8.0.0"@@ -8772,6 +8939,27 @@ __metadata:languageName: nodelinkType: hard+"hast-util-to-string@npm:^3.0.0, hast-util-to-string@npm:^3.0.1":+ version: 3.0.1+ resolution: "hast-util-to-string@npm:3.0.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ checksum: 10c0/b5fa1912a6ba6131affae52a0f4394406c4c0d23c2b0307f1d69988f1030c7bb830289303e67c5ad8f674f5f23a454c1dcd492c39e45a22c1f46d3c9bce5bd0c+ languageName: node+ linkType: hard++"hast-util-to-text@npm:^4.0.0":+ version: 4.0.2+ resolution: "hast-util-to-text@npm:4.0.2"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ "@types/unist": "npm:^3.0.0"+ hast-util-is-element: "npm:^3.0.0"+ unist-util-find-after: "npm:^5.0.0"+ checksum: 10c0/93ecc10e68fe5391c6e634140eb330942e71dea2724c8e0c647c73ed74a8ec930a4b77043b5081284808c96f73f2bee64ee416038ece75a63a467e8d14f09946+ languageName: node+ linkType: hard+"hast-util-whitespace@npm:^3.0.0":version: 3.0.0resolution: "hast-util-whitespace@npm:3.0.0"@@ -11686,7 +11874,7 @@ __metadata:languageName: nodelinkType: hard-"nth-check@npm:^2.0.1":+"nth-check@npm:^2.0.0, nth-check@npm:^2.0.1":version: 2.1.1resolution: "nth-check@npm:2.1.1"dependencies:@@ -13740,6 +13928,27 @@ __metadata:languageName: nodelinkType: hard+"rehype-minify-whitespace@npm:^6.0.0":+ version: 6.0.2+ resolution: "rehype-minify-whitespace@npm:6.0.2"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ hast-util-minify-whitespace: "npm:^1.0.0"+ checksum: 10c0/e808a452068392070dcba4ea0fdc24c783e21ddc9c70008f90827ddd29afa6fb82f77473bba91e06b48cef8575553f906fa8ab44ae59700f945eb0910927acd9+ languageName: node+ linkType: hard++"rehype-parse@npm:^9":+ version: 9.0.1+ resolution: "rehype-parse@npm:9.0.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ hast-util-from-html: "npm:^2.0.0"+ unified: "npm:^11.0.0"+ checksum: 10c0/efa9ca17673fe70e2d322a1d262796bbed5f6a89382f8f8393352bbd6f6bbf1d4d1d050984b86ff9cb6c0fa2535175ab0829e53c94b1e38fc3c158e6c0ad90bc+ languageName: node+ linkType: hard+"rehype-raw@npm:^7.0.0":version: 7.0.0resolution: "rehype-raw@npm:7.0.0"@@ -13762,6 +13971,19 @@ __metadata:languageName: nodelinkType: hard+"rehype-remark@npm:^10":+ version: 10.0.1+ resolution: "rehype-remark@npm:10.0.1"+ dependencies:+ "@types/hast": "npm:^3.0.0"+ "@types/mdast": "npm:^4.0.0"+ hast-util-to-mdast: "npm:^10.0.0"+ unified: "npm:^11.0.0"+ vfile: "npm:^6.0.0"+ checksum: 10c0/e013fad22dd7b3bf653a79cf3dc4fecd434c5eb5f89f41e1932ae12f592b3a83c980f759d4a6ae764a61a6ea7f08330f9908c235c11510d3be731e80290aa0ba+ languageName: node+ linkType: hard+"relateurl@npm:^0.2.7":version: 0.2.7resolution: "relateurl@npm:0.2.7"@@ -13806,7 +14028,7 @@ __metadata:languageName: nodelinkType: hard-"remark-gfm@npm:^4.0.0":+"remark-gfm@npm:^4, remark-gfm@npm:^4.0.0":version: 4.0.1resolution: "remark-gfm@npm:4.0.1"dependencies:@@ -13855,7 +14077,7 @@ __metadata:languageName: nodelinkType: hard-"remark-stringify@npm:^11.0.0":+"remark-stringify@npm:^11, remark-stringify@npm:^11.0.0":version: 11.0.0resolution: "remark-stringify@npm:11.0.0"dependencies:@@ -14090,6 +14312,7 @@ __metadata:"@docusaurus/types": "npm:3.8.1""@giscus/react": "npm:^3.0.0""@mdx-js/react": "npm:^3.0.1"+ "@signalwire/docusaurus-plugin-llms-txt": "npm:^1.2.1""@types/react": "npm:^19.0.0""@typescript-eslint/eslint-plugin": "npm:^7.0.0""@typescript-eslint/parser": "npm:^7.0.0"@@ -14831,7 +15054,7 @@ __metadata:languageName: nodelinkType: hard-"string-width@npm:^5.0.1, string-width@npm:^5.1.2":+"string-width@npm:^5.0.0, string-width@npm:^5.0.1, string-width@npm:^5.1.2":version: 5.1.2resolution: "string-width@npm:5.1.2"dependencies:@@ -15257,6 +15480,13 @@ __metadata:languageName: nodelinkType: hard+"trim-trailing-lines@npm:^2.0.0":+ version: 2.1.0+ resolution: "trim-trailing-lines@npm:2.1.0"+ checksum: 10c0/9b010d16b191422d08678f5a4988213dffd8ae9445e1b0f7f7b3e5b28ffdb062a8465a7988b66999b90589b386ddc93b56d23545ba75a74ebaf5838b30594cb9+ languageName: node+ linkType: hard+"trough@npm:^2.0.0":version: 2.2.0resolution: "trough@npm:2.2.0"@@ -15502,7 +15732,7 @@ __metadata:languageName: nodelinkType: hard-"unified@npm:^11.0.0, unified@npm:^11.0.3, unified@npm:^11.0.4":+"unified@npm:^11, unified@npm:^11.0.0, unified@npm:^11.0.3, unified@npm:^11.0.4":version: 11.0.5resolution: "unified@npm:11.0.5"dependencies:@@ -15544,6 +15774,16 @@ __metadata:languageName: nodelinkType: hard+"unist-util-find-after@npm:^5.0.0":+ version: 5.0.0+ resolution: "unist-util-find-after@npm:5.0.0"+ dependencies:+ "@types/unist": "npm:^3.0.0"+ unist-util-is: "npm:^6.0.0"+ checksum: 10c0/a7cea473c4384df8de867c456b797ff1221b20f822e1af673ff5812ed505358b36f47f3b084ac14c3622cb879ed833b71b288e8aa71025352a2aab4c2925a6eb+ languageName: node+ linkType: hard+"unist-util-is@npm:^6.0.0":version: 6.0.0resolution: "unist-util-is@npm:6.0.0"@@ -15590,7 +15830,7 @@ __metadata:languageName: nodelinkType: hard-"unist-util-visit@npm:^5.0.0":+"unist-util-visit@npm:^5, unist-util-visit@npm:^5.0.0":version: 5.0.0resolution: "unist-util-visit@npm:5.0.0"dependencies: