Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: rework navbar links to support versioning

Martin Adámek9fa4830f9038
diff --git a/docs/examples/index.mdx b/docs/examples/index.mdx
new file mode 100644
index 000000000..11f8ff124
--- /dev/null
+++ b/docs/examples/index.mdx
@@ -0,0 +1,9 @@
+---
+id: examples
+title: Examples
+---
+
+import DocCardList from '@theme/DocCardList';
+import { useCurrentSidebarCategory } from '@docusaurus/theme-common';
+
+<DocCardList items={useCurrentSidebarCategory().items} />
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index 722ee9796..bd07676d1 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -140,20 +140,22 @@ module.exports = {
},
items: [
{
- type: 'docsVersion',
- to: 'docs/quick-start',
+ type: 'doc',
+ docId: 'quick-start/quick-start',
label: 'Docs',
position: 'left',
+ activeBaseRegex: 'docs/quick-start',
},
{
- type: 'docsVersion',
- to: 'docs/examples',
+ type: 'doc',
+ docId: 'examples/examples',
label: 'Examples',
position: 'left',
+ activeBaseRegex: 'docs/examples',
},
{
- type: 'docsVersion',
- to: 'api/core',
+ type: 'custom-api',
+ to: 'core',
label: 'API',
position: 'left',
activeBaseRegex: 'api/(?!core/changelog)',
diff --git a/website/sidebars.js b/website/sidebars.js
index 2ba457076..654fdadc6 100644
--- a/website/sidebars.js
+++ b/website/sidebars.js
@@ -50,10 +50,8 @@ module.exports = {
type: 'category',
label: 'Examples',
link: {
- type: 'generated-index',
- title: 'Examples',
- slug: '/examples',
- keywords: ['examples'],
+ type: 'doc',
+ id: 'examples/examples',
},
items: [
{
diff --git a/website/src/theme/NavbarItem/ComponentTypes.js b/website/src/theme/NavbarItem/ComponentTypes.js
new file mode 100644
index 000000000..ff0e5058a
--- /dev/null
+++ b/website/src/theme/NavbarItem/ComponentTypes.js
@@ -0,0 +1,91 @@
+import DefaultNavbarItem from '@theme/NavbarItem/DefaultNavbarItem';
+import DropdownNavbarItem from '@theme/NavbarItem/DropdownNavbarItem';
+import LocaleDropdownNavbarItem from '@theme/NavbarItem/LocaleDropdownNavbarItem';
+import SearchNavbarItem from '@theme/NavbarItem/SearchNavbarItem';
+import HtmlNavbarItem from '@theme/NavbarItem/HtmlNavbarItem';
+import DocSidebarNavbarItem from '@theme/NavbarItem/DocSidebarNavbarItem';
+import DocsVersionNavbarItem from '@theme/NavbarItem/DocsVersionNavbarItem';
+import DocsVersionDropdownNavbarItem from '@theme/NavbarItem/DocsVersionDropdownNavbarItem';
+import React from 'react';
+import { useActiveDocContext } from '@docusaurus/plugin-content-docs/client';
+import { useDocsVersion, useLayoutDoc } from '@docusaurus/theme-common/internal';
+import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
+
+const pkg = require('../../../../packages/crawlee/package.json');
+
+const [v1, v2] = pkg.version.split('.');
+const stable = [v1, v2].join('.');
+
+function DocNavbarItem({
+ docId,
+ label: staticLabel,
+ docsPluginId,
+ ...props
+}) {
+ const { activeDoc } = useActiveDocContext(docsPluginId);
+ const doc = useLayoutDoc(docId, docsPluginId);
+ // Draft items are not displayed in the navbar.
+ if (doc === null) {
+ return null;
+ }
+ return (
+ <DefaultNavbarItem
+ exact
+ {...props}
+ isActive={() => activeDoc?.path.startsWith(doc.path)}
+ label={staticLabel ?? doc.id}
+ to={doc.path}
+ />
+ );
+}
+
+function ApiNavbarItem(ctx) {
+ const { activeDoc, activeVersion } = useActiveDocContext();
+ const version = useDocsVersion();
+ const { siteConfig } = useDocusaurusContext();
+ console.log(ctx, activeDoc, activeVersion, activeVersion?.path.replace(/^\/docs/, '/api'));
+ console.log(window.location.href, version);
+ // const { activeDoc } = useActiveDocContext(docsPluginId);
+ // const doc = useLayoutDoc(docId, docsPluginId);
+ // console.log(activeDoc, doc, (!!activeDoc?.sidebar && activeDoc.sidebar === doc.sidebar));
+ // Draft items are not displayed in the navbar.
+ // if (doc === null) {
+ // return null;
+ // }
+
+ if (siteConfig.presets[0][1].docs.disableVersioning || version.version === stable) {
+ return (
+ <DefaultNavbarItem
+ exact
+ {...ctx}
+ isActive={() => activeDoc?.path.startsWith(activeVersion?.path.replace(/^\/docs/, '/api'))}
+ label={ctx.staticLabel ?? ctx.label}
+ to={`api/${ctx.to}`}
+ />
+ );
+ }
+
+ return (
+ <DefaultNavbarItem
+ exact
+ {...ctx}
+ isActive={() => activeDoc?.path.startsWith(activeVersion?.path.replace(/^\/docs/, '/api'))}
+ label={ctx.staticLabel ?? ctx.label}
+ to={`api/${version.version === 'current' ? 'next' : version.version}/${ctx.to}`}
+ />
+ );
+}
+
+const ComponentTypes = {
+ 'default': DefaultNavbarItem,
+ 'localeDropdown': LocaleDropdownNavbarItem,
+ 'search': SearchNavbarItem,
+ 'dropdown': DropdownNavbarItem,
+ 'html': HtmlNavbarItem,
+ 'custom-api': ApiNavbarItem,
+ 'doc': DocNavbarItem,
+ 'docSidebar': DocSidebarNavbarItem,
+ 'docsVersion': DocsVersionNavbarItem,
+ 'docsVersionDropdown': DocsVersionDropdownNavbarItem,
+};
+export default ComponentTypes;
diff --git a/website/static/js/custom.js b/website/static/js/custom.js
index 19c7f3aa6..10f43f191 100644
--- a/website/static/js/custom.js
+++ b/website/static/js/custom.js
@@ -6,7 +6,12 @@ function load() {
for (const el of versions) {
const match = el.href.match(/\/docs\/(\d+\.\d+(\.\d+)?)$/) || el.href.match(/\/docs\/(\d+\.\d+(\.\d+)?)/);
- const version = (types[i++] || match[0] || '').replace('/docs', '/api');
+
+ if (!types[i++] && !match) {
+ continue;
+ }
+
+ const version = (types[i++] || match[0]).replace('/docs', '/api');
if (el.classList.contains('api-version-bound')) {
continue;
@@ -20,6 +25,24 @@ function load() {
});
el.classList.add('api-version-bound');
}
+
+ // const navbarLinks = document.querySelectorAll('.navbar a.navbar__link');
+ //
+ // for (const el of [...navbarLinks].slice(1, 3)) {
+ // if (el.classList.contains('api-version-bound')) {
+ // continue;
+ // }
+ //
+ // const url = new URL(el.href);
+ // const parts = url.pathname.split('/');
+ // parts.splice(2, 0, 'next');
+ // url.pathname = parts.join('/');
+ // el.href = url.href;
+ // el.classList.add('api-version-bound');
+ // el.addEventListener('click', (e) => {
+ // e.stopPropagation();
+ // });
+ // }
}
setInterval(() => {
diff --git a/website/versioned_docs/version-3.0/examples/index.mdx b/website/versioned_docs/version-3.0/examples/index.mdx
new file mode 100644
index 000000000..11f8ff124
--- /dev/null
+++ b/website/versioned_docs/version-3.0/examples/index.mdx
@@ -0,0 +1,9 @@
+---
+id: examples
+title: Examples
+---
+
+import DocCardList from '@theme/DocCardList';
+import { useCurrentSidebarCategory } from '@docusaurus/theme-common';
+
+<DocCardList items={useCurrentSidebarCategory().items} />
diff --git a/website/versioned_sidebars/version-3.0-sidebars.json b/website/versioned_sidebars/version-3.0-sidebars.json
index 877b43d16..011a4defa 100644
--- a/website/versioned_sidebars/version-3.0-sidebars.json
+++ b/website/versioned_sidebars/version-3.0-sidebars.json
@@ -52,12 +52,8 @@
"type": "category",
"label": "Examples",
"link": {
- "type": "generated-index",
- "title": "Examples",
- "slug": "/examples",
- "keywords": [
- "examples"
- ]
+ "type": "doc",
+ "id": "examples/examples"
},
"items": [
{