Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: add giscus discussions to docs pages
Martin Adámek3f09eb18ae9a
diff --git a/website/package-lock.json b/website/package-lock.jsonindex b4875ab3b..bfdf0142e 100644--- a/website/package-lock.json+++ b/website/package-lock.json@@ -8,6 +8,7 @@"@docusaurus/core": "2.0.1","@docusaurus/plugin-client-redirects": "2.0.1","@docusaurus/preset-classic": "2.0.1",+ "@giscus/react": "^2.2.0","clsx": "^1.1.1","docusaurus-gtm-plugin": "^0.0.2","docusaurus-plugin-typedoc-api": "2.3.0",@@ -2709,6 +2710,18 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/@giscus/react": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/@giscus/react/-/react-2.2.0.tgz",+ "integrity": "sha512-MXjKAQyO85ZdC8OVApNQsJhPvTJcuUf5GmQw2KdvNcBVivUpVrA+NUppdwsrI+lK6jEX4ZMVOCnbJ5GaQddaAA==",+ "dependencies": {+ "giscus": "^1.2.0"+ },+ "peerDependencies": {+ "react": "^16 || ^17 || ^18",+ "react-dom": "^16 || ^17 || ^18"+ }+ },"node_modules/@hapi/hoek": {"version": "9.3.0","resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",@@ -2811,6 +2824,11 @@"resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz","integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A=="},+ "node_modules/@lit/reactive-element": {+ "version": "1.3.4",+ "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-1.3.4.tgz",+ "integrity": "sha512-I1wz4uxOA52zSBhKmv4KQWLJpCyvfpnDg+eQR6mjpRgV+Ldi14HLPpSUpJklZRldz0fFmGCC/kVmuc/3cPFqCg=="+ },"node_modules/@mdx-js/mdx": {"version": "1.6.22","resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-1.6.22.tgz",@@ -3586,6 +3604,11 @@"@types/node": "*"}},+ "node_modules/@types/trusted-types": {+ "version": "2.0.2",+ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz",+ "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg=="+ },"node_modules/@types/unist": {"version": "2.0.6","resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",@@ -7446,6 +7469,14 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/giscus": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/giscus/-/giscus-1.2.0.tgz",+ "integrity": "sha512-IpfWvU0/hYbMGQKuoPlED8wWmluRYIOjtrBCnL7logsWjMpPRxiAC2pUIC0+SC0pDMOqXrk1onTYMHgwgRpRzg==",+ "dependencies": {+ "lit": "^2.2.8"+ }+ },"node_modules/github-slugger": {"version": "1.4.0","resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.4.0.tgz",@@ -8932,6 +8963,33 @@"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz","integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="},+ "node_modules/lit": {+ "version": "2.2.8",+ "resolved": "https://registry.npmjs.org/lit/-/lit-2.2.8.tgz",+ "integrity": "sha512-QjeNbi/H9LVIHR+u0OqsL+hs62a16m02JlJHYN48HcBuXyiPYR8JvzsTp5dYYS81l+b9Emp3UaGo82EheV0pog==",+ "dependencies": {+ "@lit/reactive-element": "^1.3.0",+ "lit-element": "^3.2.0",+ "lit-html": "^2.2.0"+ }+ },+ "node_modules/lit-element": {+ "version": "3.2.2",+ "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-3.2.2.tgz",+ "integrity": "sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==",+ "dependencies": {+ "@lit/reactive-element": "^1.3.0",+ "lit-html": "^2.2.0"+ }+ },+ "node_modules/lit-html": {+ "version": "2.2.7",+ "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-2.2.7.tgz",+ "integrity": "sha512-JhqiAwO1l03kRe68uBZ0i2x4ef2S5szY9vvP411nlrFZIpKK4/hwnhA/15bqbvxe1lV3ipBdhaOzHmyOk7QIRg==",+ "dependencies": {+ "@types/trusted-types": "^2.0.2"+ }+ },"node_modules/loader-runner": {"version": "4.3.0","resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",@@ -16543,6 +16601,14 @@}}},+ "@giscus/react": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/@giscus/react/-/react-2.2.0.tgz",+ "integrity": "sha512-MXjKAQyO85ZdC8OVApNQsJhPvTJcuUf5GmQw2KdvNcBVivUpVrA+NUppdwsrI+lK6jEX4ZMVOCnbJ5GaQddaAA==",+ "requires": {+ "giscus": "^1.2.0"+ }+ },"@hapi/hoek": {"version": "9.3.0","resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",@@ -16632,6 +16698,11 @@"resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz","integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A=="},+ "@lit/reactive-element": {+ "version": "1.3.4",+ "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-1.3.4.tgz",+ "integrity": "sha512-I1wz4uxOA52zSBhKmv4KQWLJpCyvfpnDg+eQR6mjpRgV+Ldi14HLPpSUpJklZRldz0fFmGCC/kVmuc/3cPFqCg=="+ },"@mdx-js/mdx": {"version": "1.6.22","resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-1.6.22.tgz",@@ -17220,6 +17291,11 @@"@types/node": "*"}},+ "@types/trusted-types": {+ "version": "2.0.2",+ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz",+ "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg=="+ },"@types/unist": {"version": "2.0.6","resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",@@ -20032,6 +20108,14 @@"get-intrinsic": "^1.1.1"}},+ "giscus": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/giscus/-/giscus-1.2.0.tgz",+ "integrity": "sha512-IpfWvU0/hYbMGQKuoPlED8wWmluRYIOjtrBCnL7logsWjMpPRxiAC2pUIC0+SC0pDMOqXrk1onTYMHgwgRpRzg==",+ "requires": {+ "lit": "^2.2.8"+ }+ },"github-slugger": {"version": "1.4.0","resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.4.0.tgz",@@ -21087,6 +21171,33 @@"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz","integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="},+ "lit": {+ "version": "2.2.8",+ "resolved": "https://registry.npmjs.org/lit/-/lit-2.2.8.tgz",+ "integrity": "sha512-QjeNbi/H9LVIHR+u0OqsL+hs62a16m02JlJHYN48HcBuXyiPYR8JvzsTp5dYYS81l+b9Emp3UaGo82EheV0pog==",+ "requires": {+ "@lit/reactive-element": "^1.3.0",+ "lit-element": "^3.2.0",+ "lit-html": "^2.2.0"+ }+ },+ "lit-element": {+ "version": "3.2.2",+ "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-3.2.2.tgz",+ "integrity": "sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==",+ "requires": {+ "@lit/reactive-element": "^1.3.0",+ "lit-html": "^2.2.0"+ }+ },+ "lit-html": {+ "version": "2.2.7",+ "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-2.2.7.tgz",+ "integrity": "sha512-JhqiAwO1l03kRe68uBZ0i2x4ef2S5szY9vvP411nlrFZIpKK4/hwnhA/15bqbvxe1lV3ipBdhaOzHmyOk7QIRg==",+ "requires": {+ "@types/trusted-types": "^2.0.2"+ }+ },"loader-runner": {"version": "4.3.0","resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",diff --git a/website/package.json b/website/package.jsonindex 12fdef395..bea367e4a 100644--- a/website/package.json+++ b/website/package.json@@ -31,6 +31,7 @@"@docusaurus/core": "2.0.1","@docusaurus/plugin-client-redirects": "2.0.1","@docusaurus/preset-classic": "2.0.1",+ "@giscus/react": "^2.2.0","clsx": "^1.1.1","docusaurus-gtm-plugin": "^0.0.2","docusaurus-plugin-typedoc-api": "2.3.0",diff --git a/website/src/theme/DocItem/Layout/index.js b/website/src/theme/DocItem/Layout/index.jsnew file mode 100644index 000000000..d7b4ef566--- /dev/null+++ b/website/src/theme/DocItem/Layout/index.js@@ -0,0 +1,75 @@+import React from 'react';+import clsx from 'clsx';+import { useWindowSize } from '@docusaurus/theme-common';+import { useDoc } from '@docusaurus/theme-common/internal';+import DocItemPaginator from '@theme/DocItem/Paginator';+import DocVersionBanner from '@theme/DocVersionBanner';+import DocVersionBadge from '@theme/DocVersionBadge';+import DocItemFooter from '@theme/DocItem/Footer';+import DocItemTOCMobile from '@theme/DocItem/TOC/Mobile';+import DocItemTOCDesktop from '@theme/DocItem/TOC/Desktop';+import DocItemContent from '@theme/DocItem/Content';+import DocBreadcrumbs from '@theme/DocBreadcrumbs';+import Giscus from '@giscus/react';+import styles from './styles.module.css';++/**+ * Decide if the toc should be rendered, on mobile or desktop viewports+ */+function useDocTOC() {+ const {+ frontMatter,+ toc,+ } = useDoc();+ const windowSize = useWindowSize();+ const hidden = frontMatter.hide_table_of_contents;+ const canRender = !hidden && toc.length > 0;+ const mobile = canRender ? <DocItemTOCMobile/> : undefined;+ const desktop = canRender && (windowSize === 'desktop' || windowSize === 'ssr') ? (+ <DocItemTOCDesktop/>+ ) : undefined;+ return {+ hidden,+ mobile,+ desktop,+ };+}++export default function DocItemLayout({ children }) {+ const docTOC = useDocTOC();+ return (+ <div className="row">+ <div className={clsx('col', !docTOC.hidden && styles.docItemCol)}>+ <DocVersionBanner/>+ <div className={styles.docItemContainer}>+ <article>+ <DocBreadcrumbs/>+ <DocVersionBadge/>+ {docTOC.mobile}+ <DocItemContent>{children}</DocItemContent>+ <DocItemFooter/>+ </article>+ <DocItemPaginator/>+ </div>++ <Giscus+ id="comments"+ repo="apify/crawlee"+ repoId="MDEwOlJlcG9zaXRvcnk2NjY3MDgxOQ="+ category="Comments"+ categoryId="DIC_kwDOA_lQ484CQufN"+ mapping="title"+ reactionsEnabled="1"+ emitMetadata="0"+ inputPosition="top"+ theme="preferred_color_scheme"+ lang="en"+ strict="0"+ loading="lazy"+ />+ </div>++ {docTOC.desktop && <div className="col col--3">{docTOC.desktop}</div>}+ </div>+ );+}diff --git a/website/src/theme/DocItem/Layout/styles.module.css b/website/src/theme/DocItem/Layout/styles.module.cssnew file mode 100644index 000000000..c793f7d7a--- /dev/null+++ b/website/src/theme/DocItem/Layout/styles.module.css@@ -0,0 +1,14 @@+.docItemContainer {+ margin-bottom: 50px;+}++.docItemContainer header + *,+.docItemContainer article > *:first-child {+ margin-top: 0;+}++@media (min-width: 997px) {+ .docItemCol {+ max-width: 75% !important;+ }+}