Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: add giscus discussions to docs pages

Martin Adámek3f09eb18ae9a
diff --git a/website/package-lock.json b/website/package-lock.json
index b4875ab3b..bfdf0142e 100644
--- a/website/package-lock.json
+++ b/website/package-lock.json
@@ -8,6 +8,7 @@
"@docusaurus/core": "2.0.1",
"@docusaurus/plugin-client-redirects": "2.0.1",
"@docusaurus/preset-classic": "2.0.1",
+ "@giscus/react": "^2.2.0",
"clsx": "^1.1.1",
"docusaurus-gtm-plugin": "^0.0.2",
"docusaurus-plugin-typedoc-api": "2.3.0",
@@ -2709,6 +2710,18 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/@giscus/react": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@giscus/react/-/react-2.2.0.tgz",
+ "integrity": "sha512-MXjKAQyO85ZdC8OVApNQsJhPvTJcuUf5GmQw2KdvNcBVivUpVrA+NUppdwsrI+lK6jEX4ZMVOCnbJ5GaQddaAA==",
+ "dependencies": {
+ "giscus": "^1.2.0"
+ },
+ "peerDependencies": {
+ "react": "^16 || ^17 || ^18",
+ "react-dom": "^16 || ^17 || ^18"
+ }
+ },
"node_modules/@hapi/hoek": {
"version": "9.3.0",
"resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",
@@ -2811,6 +2824,11 @@
"resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz",
"integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A=="
},
+ "node_modules/@lit/reactive-element": {
+ "version": "1.3.4",
+ "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-1.3.4.tgz",
+ "integrity": "sha512-I1wz4uxOA52zSBhKmv4KQWLJpCyvfpnDg+eQR6mjpRgV+Ldi14HLPpSUpJklZRldz0fFmGCC/kVmuc/3cPFqCg=="
+ },
"node_modules/@mdx-js/mdx": {
"version": "1.6.22",
"resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-1.6.22.tgz",
@@ -3586,6 +3604,11 @@
"@types/node": "*"
}
},
+ "node_modules/@types/trusted-types": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz",
+ "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg=="
+ },
"node_modules/@types/unist": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",
@@ -7446,6 +7469,14 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/giscus": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/giscus/-/giscus-1.2.0.tgz",
+ "integrity": "sha512-IpfWvU0/hYbMGQKuoPlED8wWmluRYIOjtrBCnL7logsWjMpPRxiAC2pUIC0+SC0pDMOqXrk1onTYMHgwgRpRzg==",
+ "dependencies": {
+ "lit": "^2.2.8"
+ }
+ },
"node_modules/github-slugger": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.4.0.tgz",
@@ -8932,6 +8963,33 @@
"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
"integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
},
+ "node_modules/lit": {
+ "version": "2.2.8",
+ "resolved": "https://registry.npmjs.org/lit/-/lit-2.2.8.tgz",
+ "integrity": "sha512-QjeNbi/H9LVIHR+u0OqsL+hs62a16m02JlJHYN48HcBuXyiPYR8JvzsTp5dYYS81l+b9Emp3UaGo82EheV0pog==",
+ "dependencies": {
+ "@lit/reactive-element": "^1.3.0",
+ "lit-element": "^3.2.0",
+ "lit-html": "^2.2.0"
+ }
+ },
+ "node_modules/lit-element": {
+ "version": "3.2.2",
+ "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-3.2.2.tgz",
+ "integrity": "sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==",
+ "dependencies": {
+ "@lit/reactive-element": "^1.3.0",
+ "lit-html": "^2.2.0"
+ }
+ },
+ "node_modules/lit-html": {
+ "version": "2.2.7",
+ "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-2.2.7.tgz",
+ "integrity": "sha512-JhqiAwO1l03kRe68uBZ0i2x4ef2S5szY9vvP411nlrFZIpKK4/hwnhA/15bqbvxe1lV3ipBdhaOzHmyOk7QIRg==",
+ "dependencies": {
+ "@types/trusted-types": "^2.0.2"
+ }
+ },
"node_modules/loader-runner": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
@@ -16543,6 +16601,14 @@
}
}
},
+ "@giscus/react": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@giscus/react/-/react-2.2.0.tgz",
+ "integrity": "sha512-MXjKAQyO85ZdC8OVApNQsJhPvTJcuUf5GmQw2KdvNcBVivUpVrA+NUppdwsrI+lK6jEX4ZMVOCnbJ5GaQddaAA==",
+ "requires": {
+ "giscus": "^1.2.0"
+ }
+ },
"@hapi/hoek": {
"version": "9.3.0",
"resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",
@@ -16632,6 +16698,11 @@
"resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz",
"integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A=="
},
+ "@lit/reactive-element": {
+ "version": "1.3.4",
+ "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-1.3.4.tgz",
+ "integrity": "sha512-I1wz4uxOA52zSBhKmv4KQWLJpCyvfpnDg+eQR6mjpRgV+Ldi14HLPpSUpJklZRldz0fFmGCC/kVmuc/3cPFqCg=="
+ },
"@mdx-js/mdx": {
"version": "1.6.22",
"resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-1.6.22.tgz",
@@ -17220,6 +17291,11 @@
"@types/node": "*"
}
},
+ "@types/trusted-types": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz",
+ "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg=="
+ },
"@types/unist": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",
@@ -20032,6 +20108,14 @@
"get-intrinsic": "^1.1.1"
}
},
+ "giscus": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/giscus/-/giscus-1.2.0.tgz",
+ "integrity": "sha512-IpfWvU0/hYbMGQKuoPlED8wWmluRYIOjtrBCnL7logsWjMpPRxiAC2pUIC0+SC0pDMOqXrk1onTYMHgwgRpRzg==",
+ "requires": {
+ "lit": "^2.2.8"
+ }
+ },
"github-slugger": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.4.0.tgz",
@@ -21087,6 +21171,33 @@
"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
"integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
},
+ "lit": {
+ "version": "2.2.8",
+ "resolved": "https://registry.npmjs.org/lit/-/lit-2.2.8.tgz",
+ "integrity": "sha512-QjeNbi/H9LVIHR+u0OqsL+hs62a16m02JlJHYN48HcBuXyiPYR8JvzsTp5dYYS81l+b9Emp3UaGo82EheV0pog==",
+ "requires": {
+ "@lit/reactive-element": "^1.3.0",
+ "lit-element": "^3.2.0",
+ "lit-html": "^2.2.0"
+ }
+ },
+ "lit-element": {
+ "version": "3.2.2",
+ "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-3.2.2.tgz",
+ "integrity": "sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==",
+ "requires": {
+ "@lit/reactive-element": "^1.3.0",
+ "lit-html": "^2.2.0"
+ }
+ },
+ "lit-html": {
+ "version": "2.2.7",
+ "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-2.2.7.tgz",
+ "integrity": "sha512-JhqiAwO1l03kRe68uBZ0i2x4ef2S5szY9vvP411nlrFZIpKK4/hwnhA/15bqbvxe1lV3ipBdhaOzHmyOk7QIRg==",
+ "requires": {
+ "@types/trusted-types": "^2.0.2"
+ }
+ },
"loader-runner": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
diff --git a/website/package.json b/website/package.json
index 12fdef395..bea367e4a 100644
--- a/website/package.json
+++ b/website/package.json
@@ -31,6 +31,7 @@
"@docusaurus/core": "2.0.1",
"@docusaurus/plugin-client-redirects": "2.0.1",
"@docusaurus/preset-classic": "2.0.1",
+ "@giscus/react": "^2.2.0",
"clsx": "^1.1.1",
"docusaurus-gtm-plugin": "^0.0.2",
"docusaurus-plugin-typedoc-api": "2.3.0",
diff --git a/website/src/theme/DocItem/Layout/index.js b/website/src/theme/DocItem/Layout/index.js
new file mode 100644
index 000000000..d7b4ef566
--- /dev/null
+++ b/website/src/theme/DocItem/Layout/index.js
@@ -0,0 +1,75 @@
+import React from 'react';
+import clsx from 'clsx';
+import { useWindowSize } from '@docusaurus/theme-common';
+import { useDoc } from '@docusaurus/theme-common/internal';
+import DocItemPaginator from '@theme/DocItem/Paginator';
+import DocVersionBanner from '@theme/DocVersionBanner';
+import DocVersionBadge from '@theme/DocVersionBadge';
+import DocItemFooter from '@theme/DocItem/Footer';
+import DocItemTOCMobile from '@theme/DocItem/TOC/Mobile';
+import DocItemTOCDesktop from '@theme/DocItem/TOC/Desktop';
+import DocItemContent from '@theme/DocItem/Content';
+import DocBreadcrumbs from '@theme/DocBreadcrumbs';
+import Giscus from '@giscus/react';
+import styles from './styles.module.css';
+
+/**
+ * Decide if the toc should be rendered, on mobile or desktop viewports
+ */
+function useDocTOC() {
+ const {
+ frontMatter,
+ toc,
+ } = useDoc();
+ const windowSize = useWindowSize();
+ const hidden = frontMatter.hide_table_of_contents;
+ const canRender = !hidden && toc.length > 0;
+ const mobile = canRender ? <DocItemTOCMobile/> : undefined;
+ const desktop = canRender && (windowSize === 'desktop' || windowSize === 'ssr') ? (
+ <DocItemTOCDesktop/>
+ ) : undefined;
+ return {
+ hidden,
+ mobile,
+ desktop,
+ };
+}
+
+export default function DocItemLayout({ children }) {
+ const docTOC = useDocTOC();
+ return (
+ <div className="row">
+ <div className={clsx('col', !docTOC.hidden && styles.docItemCol)}>
+ <DocVersionBanner/>
+ <div className={styles.docItemContainer}>
+ <article>
+ <DocBreadcrumbs/>
+ <DocVersionBadge/>
+ {docTOC.mobile}
+ <DocItemContent>{children}</DocItemContent>
+ <DocItemFooter/>
+ </article>
+ <DocItemPaginator/>
+ </div>
+
+ <Giscus
+ id="comments"
+ repo="apify/crawlee"
+ repoId="MDEwOlJlcG9zaXRvcnk2NjY3MDgxOQ="
+ category="Comments"
+ categoryId="DIC_kwDOA_lQ484CQufN"
+ mapping="title"
+ reactionsEnabled="1"
+ emitMetadata="0"
+ inputPosition="top"
+ theme="preferred_color_scheme"
+ lang="en"
+ strict="0"
+ loading="lazy"
+ />
+ </div>
+
+ {docTOC.desktop && <div className="col col--3">{docTOC.desktop}</div>}
+ </div>
+ );
+}
diff --git a/website/src/theme/DocItem/Layout/styles.module.css b/website/src/theme/DocItem/Layout/styles.module.css
new file mode 100644
index 000000000..c793f7d7a
--- /dev/null
+++ b/website/src/theme/DocItem/Layout/styles.module.css
@@ -0,0 +1,14 @@
+.docItemContainer {
+ margin-bottom: 50px;
+}
+
+.docItemContainer header + *,
+.docItemContainer article > *:first-child {
+ margin-top: 0;
+}
+
+@media (min-width: 997px) {
+ .docItemCol {
+ max-width: 75% !important;
+ }
+}