Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: Add llms.txt and llms-full.txt generator (#3061)

Jan Turoň79f3ba529f03
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index a054a387a..8d888b151 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -172,6 +172,20 @@ module.exports = {
id: 'GTM-5P7MCS7',
},
],
+ [
+ '@signalwire/docusaurus-plugin-llms-txt',
+ {
+ enableDescriptions: false,
+ content: {
+ excludeRoutes: ['/js/api/3.*/**', '/js/api/3.*', '/js/api/next/**', '/js/api/next'],
+ includeVersionedDocs: false,
+ enableLlmsFullTxt: true,
+ includeBlog: true,
+ includePages: true,
+ relativePaths: false,
+ },
+ }
+ ],
async function runnableCodeBlock() {
return {
name: 'runnable-code-block',
diff --git a/website/package.json b/website/package.json
index c93d591cb..53417495d 100644
--- a/website/package.json
+++ b/website/package.json
@@ -1,6 +1,7 @@
{
"scripts": {
"examples": "docusaurus-examples",
+ "postbuild": "sed -i '' 's/\\.md//g' build/llms.txt build/llms-full.txt",
"postinstall": "npx patch-package",
"start": "rimraf .docusaurus && docusaurus start",
"start:fast": "rimraf .docusaurus && CRAWLEE_DOCS_FAST=1 docusaurus start",
@@ -44,6 +45,7 @@
"@docusaurus/theme-common": "3.8.1",
"@giscus/react": "^3.0.0",
"@mdx-js/react": "^3.0.1",
+ "@signalwire/docusaurus-plugin-llms-txt": "^1.2.1",
"axios": "^1.5.0",
"buffer": "^6.0.3",
"clsx": "^2.0.0",
diff --git a/website/yarn.lock b/website/yarn.lock
index b4b7ee1ba..77c773022 100644
--- a/website/yarn.lock
+++ b/website/yarn.lock
@@ -3438,6 +3438,28 @@ __metadata:
languageName: node
linkType: hard
+"@signalwire/docusaurus-plugin-llms-txt@npm:^1.2.1":
+ version: 1.2.1
+ resolution: "@signalwire/docusaurus-plugin-llms-txt@npm:1.2.1"
+ dependencies:
+ fs-extra: "npm:^11.0.0"
+ hast-util-select: "npm:^6.0.4"
+ hast-util-to-html: "npm:^9.0.5"
+ hast-util-to-string: "npm:^3.0.1"
+ p-map: "npm:^7.0.2"
+ rehype-parse: "npm:^9"
+ rehype-remark: "npm:^10"
+ remark-gfm: "npm:^4"
+ remark-stringify: "npm:^11"
+ string-width: "npm:^5.0.0"
+ unified: "npm:^11"
+ unist-util-visit: "npm:^5"
+ peerDependencies:
+ "@docusaurus/core": ^3.0.0
+ checksum: 10c0/a9847760594f6ae2274c5130225e7dfc3220dd85ffefbf33ba0086a98649732b09409a1607a5b4ba814102413eb0d5a2d2365405f38bb89d7f68e34166acbebd
+ languageName: node
+ linkType: hard
+
"@sinclair/typebox@npm:^0.27.8":
version: 0.27.8
resolution: "@sinclair/typebox@npm:0.27.8"
@@ -5350,6 +5372,13 @@ __metadata:
languageName: node
linkType: hard
+"bcp-47-match@npm:^2.0.0":
+ version: 2.0.3
+ resolution: "bcp-47-match@npm:2.0.3"
+ checksum: 10c0/ae5c202854df8a9ad4777dc3b49562578495a69164869f365a88c1a089837a9fbbce4c0c44f6f1a5e44c7841f47e91fe6fea00306ca49ce5ec95a7eb71f839c4
+ languageName: node
+ linkType: hard
+
"big.js@npm:^5.2.2":
version: 5.2.2
resolution: "big.js@npm:5.2.2"
@@ -6417,6 +6446,13 @@ __metadata:
languageName: node
linkType: hard
+"css-selector-parser@npm:^3.0.0":
+ version: 3.1.3
+ resolution: "css-selector-parser@npm:3.1.3"
+ checksum: 10c0/0bba96edfd27827d79933b113c42bec627b96a79f6fe4b12dec12da109d0b3a25f2f76d385b7c28ff22dca68840251751d1061d9226657755430e4787bf4594e
+ languageName: node
+ linkType: hard
+
"css-tree@npm:^2.3.1":
version: 2.3.1
resolution: "css-tree@npm:2.3.1"
@@ -6816,6 +6852,15 @@ __metadata:
languageName: node
linkType: hard
+"direction@npm:^2.0.0":
+ version: 2.0.1
+ resolution: "direction@npm:2.0.1"
+ bin:
+ direction: cli.js
+ checksum: 10c0/dce809431cad978e0778769a3818ea797ebe0bd542c85032ad9ad98971e2021a146be62feb259d7ffe4b76739e07b23e861b29c3f184ac8d38cc6ba956d5c586
+ languageName: node
+ linkType: hard
+
"dns-packet@npm:^5.2.2":
version: 5.6.1
resolution: "dns-packet@npm:5.6.1"
@@ -8155,7 +8200,7 @@ __metadata:
languageName: node
linkType: hard
-"fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0":
+"fs-extra@npm:^11.0.0, fs-extra@npm:^11.1.0, fs-extra@npm:^11.1.1, fs-extra@npm:^11.2.0":
version: 11.3.0
resolution: "fs-extra@npm:11.3.0"
dependencies:
@@ -8645,6 +8690,30 @@ __metadata:
languageName: node
linkType: hard
+"hast-util-embedded@npm:^3.0.0":
+ version: 3.0.0
+ resolution: "hast-util-embedded@npm:3.0.0"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ hast-util-is-element: "npm:^3.0.0"
+ checksum: 10c0/054c3d3b96fcd5c1d1c6f8d38ce1f7f33022ba6362129a022673d0b539f876acdcababbb9df29812fb927294f98ef7a2f44519a80d637fe3eea1819c9e69eeac
+ languageName: node
+ linkType: hard
+
+"hast-util-from-html@npm:^2.0.0":
+ version: 2.0.3
+ resolution: "hast-util-from-html@npm:2.0.3"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ devlop: "npm:^1.1.0"
+ hast-util-from-parse5: "npm:^8.0.0"
+ parse5: "npm:^7.0.0"
+ vfile: "npm:^6.0.0"
+ vfile-message: "npm:^4.0.0"
+ checksum: 10c0/993ef707c1a12474c8d4094fc9706a72826c660a7e308ea54c50ad893353d32e139b7cbc67510c2e82feac572b320e3b05aeb13d0f9c6302d61261f337b46764
+ languageName: node
+ linkType: hard
+
"hast-util-from-parse5@npm:^8.0.0":
version: 8.0.3
resolution: "hast-util-from-parse5@npm:8.0.3"
@@ -8661,6 +8730,46 @@ __metadata:
languageName: node
linkType: hard
+"hast-util-has-property@npm:^3.0.0":
+ version: 3.0.0
+ resolution: "hast-util-has-property@npm:3.0.0"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ checksum: 10c0/6e2c0e22ca893c6ebb60f8390e184c4deb041c36d09796756f02cd121c1789c0f5c862ed06caea8f1a80ea8c0ef6a7854dd57946c2eebb76488727bd4a1c952e
+ languageName: node
+ linkType: hard
+
+"hast-util-is-body-ok-link@npm:^3.0.0":
+ version: 3.0.1
+ resolution: "hast-util-is-body-ok-link@npm:3.0.1"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ checksum: 10c0/c320cbd9a9a834b007a6f2f8c271e98b8331c0193adf06e0a7c5ea0acae664e97ce28eb4436e0658bc5cdb8f47390ec1c6cba7c4fe1ded10951fcdd1432f60bf
+ languageName: node
+ linkType: hard
+
+"hast-util-is-element@npm:^3.0.0":
+ version: 3.0.0
+ resolution: "hast-util-is-element@npm:3.0.0"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ checksum: 10c0/f5361e4c9859c587ca8eb0d8343492f3077ccaa0f58a44cd09f35d5038f94d65152288dcd0c19336ef2c9491ec4d4e45fde2176b05293437021570aa0bc3613b
+ languageName: node
+ linkType: hard
+
+"hast-util-minify-whitespace@npm:^1.0.0":
+ version: 1.0.1
+ resolution: "hast-util-minify-whitespace@npm:1.0.1"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ hast-util-embedded: "npm:^3.0.0"
+ hast-util-is-element: "npm:^3.0.0"
+ hast-util-whitespace: "npm:^3.0.0"
+ unist-util-is: "npm:^6.0.0"
+ checksum: 10c0/20a7d64947e080463084f444ad09c7f28c40e7648ca2d9c6c036e42a67f8e945d352560ff599304c988257c1e477abcf6a1f508c0900211fa58ec1ba21b36533
+ languageName: node
+ linkType: hard
+
"hast-util-parse-selector@npm:^4.0.0":
version: 4.0.0
resolution: "hast-util-parse-selector@npm:4.0.0"
@@ -8670,6 +8779,19 @@ __metadata:
languageName: node
linkType: hard
+"hast-util-phrasing@npm:^3.0.0":
+ version: 3.0.1
+ resolution: "hast-util-phrasing@npm:3.0.1"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ hast-util-embedded: "npm:^3.0.0"
+ hast-util-has-property: "npm:^3.0.0"
+ hast-util-is-body-ok-link: "npm:^3.0.0"
+ hast-util-is-element: "npm:^3.0.0"
+ checksum: 10c0/d77e186ea3d7d62f6db9c4a55c3e6d9f1f6affd5f40250e8de9d73f167ae19fcc02fafe1601dfbe36e90f76ed5013ac004f0b6b398aee3a04a7a81de12788600
+ languageName: node
+ linkType: hard
+
"hast-util-raw@npm:^9.0.0":
version: 9.1.0
resolution: "hast-util-raw@npm:9.1.0"
@@ -8691,6 +8813,29 @@ __metadata:
languageName: node
linkType: hard
+"hast-util-select@npm:^6.0.4":
+ version: 6.0.4
+ resolution: "hast-util-select@npm:6.0.4"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ "@types/unist": "npm:^3.0.0"
+ bcp-47-match: "npm:^2.0.0"
+ comma-separated-tokens: "npm:^2.0.0"
+ css-selector-parser: "npm:^3.0.0"
+ devlop: "npm:^1.0.0"
+ direction: "npm:^2.0.0"
+ hast-util-has-property: "npm:^3.0.0"
+ hast-util-to-string: "npm:^3.0.0"
+ hast-util-whitespace: "npm:^3.0.0"
+ nth-check: "npm:^2.0.0"
+ property-information: "npm:^7.0.0"
+ space-separated-tokens: "npm:^2.0.0"
+ unist-util-visit: "npm:^5.0.0"
+ zwitch: "npm:^2.0.0"
+ checksum: 10c0/d6829953f829c24ffe465c2b156f6a7cd352f7d9b4d601e0e6ca38b85cc4a720bb9f027d34881c3b2a05f4b55c9375e256dbf43ca88604230da784e1c9c7d03f
+ languageName: node
+ linkType: hard
+
"hast-util-to-estree@npm:^3.0.0":
version: 3.1.3
resolution: "hast-util-to-estree@npm:3.1.3"
@@ -8715,7 +8860,7 @@ __metadata:
languageName: node
linkType: hard
-"hast-util-to-html@npm:^9.0.4":
+"hast-util-to-html@npm:^9.0.0, hast-util-to-html@npm:^9.0.4, hast-util-to-html@npm:^9.0.5":
version: 9.0.5
resolution: "hast-util-to-html@npm:9.0.5"
dependencies:
@@ -8757,6 +8902,28 @@ __metadata:
languageName: node
linkType: hard
+"hast-util-to-mdast@npm:^10.0.0":
+ version: 10.1.2
+ resolution: "hast-util-to-mdast@npm:10.1.2"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ "@types/mdast": "npm:^4.0.0"
+ "@ungap/structured-clone": "npm:^1.0.0"
+ hast-util-phrasing: "npm:^3.0.0"
+ hast-util-to-html: "npm:^9.0.0"
+ hast-util-to-text: "npm:^4.0.0"
+ hast-util-whitespace: "npm:^3.0.0"
+ mdast-util-phrasing: "npm:^4.0.0"
+ mdast-util-to-hast: "npm:^13.0.0"
+ mdast-util-to-string: "npm:^4.0.0"
+ rehype-minify-whitespace: "npm:^6.0.0"
+ trim-trailing-lines: "npm:^2.0.0"
+ unist-util-position: "npm:^5.0.0"
+ unist-util-visit: "npm:^5.0.0"
+ checksum: 10c0/2edd4521b147734078d66e03cd43c571a0a3aeefd3fcc34659c783b25e9222ddb5c8c759b12a86ebc70a25b3888505dc59b913ff36ae17cca04d52050592a963
+ languageName: node
+ linkType: hard
+
"hast-util-to-parse5@npm:^8.0.0":
version: 8.0.0
resolution: "hast-util-to-parse5@npm:8.0.0"
@@ -8772,6 +8939,27 @@ __metadata:
languageName: node
linkType: hard
+"hast-util-to-string@npm:^3.0.0, hast-util-to-string@npm:^3.0.1":
+ version: 3.0.1
+ resolution: "hast-util-to-string@npm:3.0.1"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ checksum: 10c0/b5fa1912a6ba6131affae52a0f4394406c4c0d23c2b0307f1d69988f1030c7bb830289303e67c5ad8f674f5f23a454c1dcd492c39e45a22c1f46d3c9bce5bd0c
+ languageName: node
+ linkType: hard
+
+"hast-util-to-text@npm:^4.0.0":
+ version: 4.0.2
+ resolution: "hast-util-to-text@npm:4.0.2"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ "@types/unist": "npm:^3.0.0"
+ hast-util-is-element: "npm:^3.0.0"
+ unist-util-find-after: "npm:^5.0.0"
+ checksum: 10c0/93ecc10e68fe5391c6e634140eb330942e71dea2724c8e0c647c73ed74a8ec930a4b77043b5081284808c96f73f2bee64ee416038ece75a63a467e8d14f09946
+ languageName: node
+ linkType: hard
+
"hast-util-whitespace@npm:^3.0.0":
version: 3.0.0
resolution: "hast-util-whitespace@npm:3.0.0"
@@ -11686,7 +11874,7 @@ __metadata:
languageName: node
linkType: hard
-"nth-check@npm:^2.0.1":
+"nth-check@npm:^2.0.0, nth-check@npm:^2.0.1":
version: 2.1.1
resolution: "nth-check@npm:2.1.1"
dependencies:
@@ -13740,6 +13928,27 @@ __metadata:
languageName: node
linkType: hard
+"rehype-minify-whitespace@npm:^6.0.0":
+ version: 6.0.2
+ resolution: "rehype-minify-whitespace@npm:6.0.2"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ hast-util-minify-whitespace: "npm:^1.0.0"
+ checksum: 10c0/e808a452068392070dcba4ea0fdc24c783e21ddc9c70008f90827ddd29afa6fb82f77473bba91e06b48cef8575553f906fa8ab44ae59700f945eb0910927acd9
+ languageName: node
+ linkType: hard
+
+"rehype-parse@npm:^9":
+ version: 9.0.1
+ resolution: "rehype-parse@npm:9.0.1"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ hast-util-from-html: "npm:^2.0.0"
+ unified: "npm:^11.0.0"
+ checksum: 10c0/efa9ca17673fe70e2d322a1d262796bbed5f6a89382f8f8393352bbd6f6bbf1d4d1d050984b86ff9cb6c0fa2535175ab0829e53c94b1e38fc3c158e6c0ad90bc
+ languageName: node
+ linkType: hard
+
"rehype-raw@npm:^7.0.0":
version: 7.0.0
resolution: "rehype-raw@npm:7.0.0"
@@ -13762,6 +13971,19 @@ __metadata:
languageName: node
linkType: hard
+"rehype-remark@npm:^10":
+ version: 10.0.1
+ resolution: "rehype-remark@npm:10.0.1"
+ dependencies:
+ "@types/hast": "npm:^3.0.0"
+ "@types/mdast": "npm:^4.0.0"
+ hast-util-to-mdast: "npm:^10.0.0"
+ unified: "npm:^11.0.0"
+ vfile: "npm:^6.0.0"
+ checksum: 10c0/e013fad22dd7b3bf653a79cf3dc4fecd434c5eb5f89f41e1932ae12f592b3a83c980f759d4a6ae764a61a6ea7f08330f9908c235c11510d3be731e80290aa0ba
+ languageName: node
+ linkType: hard
+
"relateurl@npm:^0.2.7":
version: 0.2.7
resolution: "relateurl@npm:0.2.7"
@@ -13806,7 +14028,7 @@ __metadata:
languageName: node
linkType: hard
-"remark-gfm@npm:^4.0.0":
+"remark-gfm@npm:^4, remark-gfm@npm:^4.0.0":
version: 4.0.1
resolution: "remark-gfm@npm:4.0.1"
dependencies:
@@ -13855,7 +14077,7 @@ __metadata:
languageName: node
linkType: hard
-"remark-stringify@npm:^11.0.0":
+"remark-stringify@npm:^11, remark-stringify@npm:^11.0.0":
version: 11.0.0
resolution: "remark-stringify@npm:11.0.0"
dependencies:
@@ -14090,6 +14312,7 @@ __metadata:
"@docusaurus/types": "npm:3.8.1"
"@giscus/react": "npm:^3.0.0"
"@mdx-js/react": "npm:^3.0.1"
+ "@signalwire/docusaurus-plugin-llms-txt": "npm:^1.2.1"
"@types/react": "npm:^19.0.0"
"@typescript-eslint/eslint-plugin": "npm:^7.0.0"
"@typescript-eslint/parser": "npm:^7.0.0"
@@ -14831,7 +15054,7 @@ __metadata:
languageName: node
linkType: hard
-"string-width@npm:^5.0.1, string-width@npm:^5.1.2":
+"string-width@npm:^5.0.0, string-width@npm:^5.0.1, string-width@npm:^5.1.2":
version: 5.1.2
resolution: "string-width@npm:5.1.2"
dependencies:
@@ -15257,6 +15480,13 @@ __metadata:
languageName: node
linkType: hard
+"trim-trailing-lines@npm:^2.0.0":
+ version: 2.1.0
+ resolution: "trim-trailing-lines@npm:2.1.0"
+ checksum: 10c0/9b010d16b191422d08678f5a4988213dffd8ae9445e1b0f7f7b3e5b28ffdb062a8465a7988b66999b90589b386ddc93b56d23545ba75a74ebaf5838b30594cb9
+ languageName: node
+ linkType: hard
+
"trough@npm:^2.0.0":
version: 2.2.0
resolution: "trough@npm:2.2.0"
@@ -15502,7 +15732,7 @@ __metadata:
languageName: node
linkType: hard
-"unified@npm:^11.0.0, unified@npm:^11.0.3, unified@npm:^11.0.4":
+"unified@npm:^11, unified@npm:^11.0.0, unified@npm:^11.0.3, unified@npm:^11.0.4":
version: 11.0.5
resolution: "unified@npm:11.0.5"
dependencies:
@@ -15544,6 +15774,16 @@ __metadata:
languageName: node
linkType: hard
+"unist-util-find-after@npm:^5.0.0":
+ version: 5.0.0
+ resolution: "unist-util-find-after@npm:5.0.0"
+ dependencies:
+ "@types/unist": "npm:^3.0.0"
+ unist-util-is: "npm:^6.0.0"
+ checksum: 10c0/a7cea473c4384df8de867c456b797ff1221b20f822e1af673ff5812ed505358b36f47f3b084ac14c3622cb879ed833b71b288e8aa71025352a2aab4c2925a6eb
+ languageName: node
+ linkType: hard
+
"unist-util-is@npm:^6.0.0":
version: 6.0.0
resolution: "unist-util-is@npm:6.0.0"
@@ -15590,7 +15830,7 @@ __metadata:
languageName: node
linkType: hard
-"unist-util-visit@npm:^5.0.0":
+"unist-util-visit@npm:^5, unist-util-visit@npm:^5.0.0":
version: 5.0.0
resolution: "unist-util-visit@npm:5.0.0"
dependencies: