Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: port docs fixes from apify-docs (#3544)

Martin Adámek863dbca10ea0
diff --git a/website/patches/@signalwire+docusaurus-plugin-llms-txt+1.2.2.patch b/website/patches/@signalwire+docusaurus-plugin-llms-txt+1.2.2.patch
new file mode 100644
index 000000000..c39ee9ed5
--- /dev/null
+++ b/website/patches/@signalwire+docusaurus-plugin-llms-txt+1.2.2.patch
@@ -0,0 +1,13 @@
+diff --git a/node_modules/@signalwire/docusaurus-plugin-llms-txt/lib/transformation/plugins/plugin-registry.js b/node_modules/@signalwire/docusaurus-plugin-llms-txt/lib/transformation/plugins/plugin-registry.js
+index f35c2df..4d8b9bd 100644
+--- a/node_modules/@signalwire/docusaurus-plugin-llms-txt/lib/transformation/plugins/plugin-registry.js
++++ b/node_modules/@signalwire/docusaurus-plugin-llms-txt/lib/transformation/plugins/plugin-registry.js
+@@ -73,7 +73,7 @@ export class PluginRegistry {
+ }
+ // Always last - converts HTML AST to Markdown AST
+ processor.use(rehypeRemark, {
+- handlers: { br: () => ({ type: 'html', value: '<br />' }) },
++ handlers: { br: () => ({ type: 'break' }) },
+ });
+ }
+ /**
diff --git a/website/roa-loader/index.js b/website/roa-loader/index.js
index d4826ad55..fc883a465 100644
--- a/website/roa-loader/index.js
+++ b/website/roa-loader/index.js
@@ -3,11 +3,13 @@ const { inspect } = require('node:util');
const { urlToRequest } = require('loader-utils');
+const SIGNING_TOKEN = process.env.APIFY_SIGNING_TOKEN;
const signingUrl = new URL('https://api.apify.com/v2/tools/encode-and-sign');
-signingUrl.searchParams.set('token', process.env.APIFY_SIGNING_TOKEN);
+signingUrl.searchParams.set('token', SIGNING_TOKEN || '');
const queue = [];
const cache = {};
let working = false;
+let warnedAboutMissingToken = false;
function hash(source) {
return createHash('sha1').update(source).digest('hex');
@@ -89,6 +91,15 @@ module.exports = async function (code) {
return { code, hash: 'fast' };
}
+ // Skip signing if token is not configured
+ if (!SIGNING_TOKEN) {
+ if (!warnedAboutMissingToken) {
+ console.warn('APIFY_SIGNING_TOKEN is not set, skipping code signing for runnable examples.');
+ warnedAboutMissingToken = true;
+ }
+ return { code, hash: 'unsigned' };
+ }
+
console.log(`Signing ${urlToRequest(this.resourcePath)}...`, { working, queue: queue.length });
const codeHash = await encodeAndSign(code);
return { code, hash: codeHash };
diff --git a/website/src/theme/DocItem/Content/index.js b/website/src/theme/DocItem/Content/index.js
index d870c4635..b93c5df52 100644
--- a/website/src/theme/DocItem/Content/index.js
+++ b/website/src/theme/DocItem/Content/index.js
@@ -1,3 +1,4 @@
+import BrowserOnly from '@docusaurus/BrowserOnly';
import { useDoc } from '@docusaurus/plugin-content-docs/client';
import { ThemeClassNames } from '@docusaurus/theme-common';
import LLMButtons from '@site/src/components/LLMButtons';
@@ -27,7 +28,7 @@ export default function DocItemContent({ children }) {
{syntheticTitle && (
<div className={styles.docItemContent}>
{syntheticTitle && <Heading as="h1">{syntheticTitle}</Heading>}
- <LLMButtons />
+ <BrowserOnly>{() => <LLMButtons />}</BrowserOnly>
</div>
)}
<MDXContent>{children}</MDXContent>