Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
/* eslint-disable global-require */
const path = require('path');
const { externalLinkProcessor } = require('./tools/utils/externalLink');
const packages = [
'core',
'browser-pool',
'basic-crawler',
'browser-crawler',
'http-crawler',
'http-client',
'cheerio-crawler',
'puppeteer-crawler',
'playwright-crawler',
'stagehand-crawler',
'fs-storage',
'utils',
'types',
'impit-client',
'got-scraping-client',
'otel',
];
const packagesOrder = [
'@crawlee/core',
'@crawlee/cheerio',
'@crawlee/playwright',
'@crawlee/puppeteer',
'@crawlee/stagehand',
'@crawlee/basic',
'@crawlee/http',
'@crawlee/http-client',
'@crawlee/browser',
'@crawlee/fs-storage',
'@crawlee/browser-pool',
'@crawlee/utils',
'@crawlee/types',
'@crawlee/impit-client',
'@crawlee/got-scraping-client',
'@crawlee/otel',
];
/** @type {Partial<import('@docusaurus/types').DocusaurusConfig>} */
module.exports = {
title: 'Crawlee for JavaScript · Build reliable crawlers. Fast.',
tagline: 'Build reliable crawlers. Fast.',
url: 'https://crawlee.dev',
baseUrl: '/',
trailingSlash: false,
organizationName: 'apify',
projectName: 'crawlee',
scripts: ['/js/custom.js'],
favicon: 'img/favicon.ico',
customFields: {
markdownOptions: {
html: true,
},
gaGtag: true,
repoUrl: 'https://github.com/apify/crawlee',
},
onBrokenLinks: 'throw',
markdown: {
mermaid: true,
hooks: {
onBrokenMarkdownLinks: 'throw',
},
},
themes: [
'@docusaurus/theme-mermaid',
],
future: {
faster: {
// ssgWorkerThreads: true,
swcJsLoader: true,
swcJsMinimizer: true,
swcHtmlMinimizer: true,
lightningCssMinimizer: true,
rspackBundler: true,
mdxCrossCompilerCache: true,
rspackPersistentCache: true,
},
v4: {
removeLegacyPostBuildHeadAttribute: true,
useCssCascadeLayers: false,
},
},
presets: /** @type {import('@docusaurus/types').PresetConfig[]} */ ([
[
'@docusaurus/preset-classic',
/** @type {import('@docusaurus/preset-classic').Options} */
({
docs: {
showLastUpdateAuthor: true,
showLastUpdateTime: true,
path: '../docs',
exclude: ['**/node_modules/**'],
routeBasePath: 'js/docs',
sidebarPath: './sidebars.js',
rehypePlugins: [externalLinkProcessor],
disableVersioning: !!process.env.CRAWLEE_DOCS_FAST,
versions: {
// drop the label once 4.0.0 is stable and the 4.0 snapshot exists
current: {
label: '4.0 (RC)',
},
},
editUrl: (doc) => {
return `https://github.com/apify/crawlee/edit/master/website/${doc.versionDocsDirPath}/${doc.docPath}`;
},
},
blog: {
blogTitle: 'Crawlee Blog - learn how to build better scrapers',
// eslint-disable-next-line max-len
blogDescription: 'Guides and tutorials on using Crawlee, the most reliable open-source web scraping and browser automation library for JavaScript and Node.js developers.',
blogSidebarTitle: 'All posts',
blogSidebarCount: 'ALL',
},
theme: {
customCss: '/src/css/custom.css',
},
}),
],
]),
headTags: [
// Intercom messenger
{
tagName: 'script',
innerHTML: `window.intercomSettings={api_base:"https://api-iam.intercom.io",app_id:"kod1r788"};`,
attributes: {},
},
// Intercom messenger
{
tagName: 'script',
innerHTML: `(function(){var w=window;var ic=w.Intercom;if(typeof ic==="function"){ic('reattach_activator');ic('update',w.intercomSettings);}else{var d=document;var i=function(){i.c(arguments);};i.q=[];i.c=function(args){i.q.push(args);};w.Intercom=i;var l=function(){var s=d.createElement('script');s.type='text/javascript';s.async=true;s.src='https://widget.intercom.io/widget/kod1r788';var x=d.getElementsByTagName('script')[0];x.parentNode.insertBefore(s,x);};if(document.readyState==='complete'){l();}else if(w.attachEvent){w.attachEvent('onload',l);}else{w.addEventListener('load',l,false);}}})()`,
attributes: {},
},
],
plugins: [
[
'@apify/docusaurus-plugin-typedoc-api',
{
projectRoot: `${__dirname}/..`,
changelogs: true,
readmes: true,
routeBasePath: 'js/api',
sortPackages: (a, b) => {
return packagesOrder.indexOf(a.packageName) - packagesOrder.indexOf(b.packageName);
},
packages: packages.map((name) => ({ path: `packages/${name}` })),
typedocOptions: {
excludeExternals: false,
},
},
],
[
'@docusaurus/plugin-client-redirects',
{
redirects: [
{
from: '/js/docs',
to: '/js/docs/quick-start',
},
{
from: '/js/docs/next',
to: '/js/docs/next/quick-start',
},
{
from: '/js/docs/guides/environment-variables',
to: '/js/docs/guides/configuration',
},
{
from: '/js/docs/guides/getting-started',
to: '/js/docs/introduction',
},
{
from: '/js/docs/guides/apify-platform',
to: '/js/docs/deployment/apify-platform',
},
{
from: '/js/docs/3.13/experiments/experiments-system-infomation-v2',
to: '/js/docs/3.13/experiments/experiments-system-information-v2',
},
{
from: '/js/docs/3.14/experiments/experiments-system-infomation-v2',
to: '/js/docs/3.14/experiments/experiments-system-information-v2',
},
{
from: '/js/docs/3.15/experiments/experiments-system-infomation-v2',
to: '/js/docs/3.15/experiments/experiments-system-information-v2',
},
{
from: '/js/docs/3.16/experiments/experiments-system-infomation-v2',
to: '/js/docs/3.16/experiments/experiments-system-information-v2',
},
],
// createRedirects(existingPath) {
// if (!existingPath.endsWith('/')) {
// return `${existingPath}/`;
// }
//
// return undefined; // Return a falsy value: no redirect created
// },
},
],
[
'docusaurus-gtm-plugin',
{
id: 'GTM-5P7MCS7',
},
],
[
'@signalwire/docusaurus-plugin-llms-txt',
{
enableDescriptions: false,
optionalLinks: [
{
title: 'Crawlee for Python llms.txt',
url: 'https://crawlee.dev/python/llms.txt',
},
{
title: 'Crawlee for Python llms-full.txt',
url: 'https://crawlee.dev/python/llms-full.txt',
}
],
content: {
excludeRoutes: ['/js/api/3.*/**', '/js/api/3.*', '/js/api/next/**', '/js/api/next'],
includeVersionedDocs: true,
enableLlmsFullTxt: true,
includeBlog: true,
includePages: true,
relativePaths: false,
},
}
],
async function runnableCodeBlock() {
return {
name: 'runnable-code-block',
configureWebpack() {
return {
resolveLoader: {
alias: {
'roa-loader': require.resolve(`${__dirname}/roa-loader/`),
},
},
};
},
};
},
// skipping svgo for animated crawlee logo
async function doNotUseSVGO() {
return {
name: 'docusaurus-svgo',
configureWebpack(config) {
// find the svg rule
const svgRule = config.module.rules.find((r) => typeof r === 'object' && r.test.toString() === '/\\.svg$/i');
// find the svgr loader
const svgrLoader = svgRule?.oneOf?.[0];
// make copy of svgr loader and disable svgo
const svgrLoaderCopy = JSON.parse(JSON.stringify(svgrLoader));
// include only animated logo
svgrLoaderCopy.include = /animated-crawlee-logo/;
// turn off svgo
svgrLoaderCopy.use[0].options.svgo = false;
// insert the copy after the original svgr loader
svgRule.oneOf.splice(1, 0, svgrLoaderCopy);
// exclude animated logo from the first svgr loader (with svgo enabled)
svgrLoader.exclude = /animated-crawlee-logo/;
return {
mergeStrategy: {
'module.rules': 'replace',
},
module: {
rules: config.module.rules,
},
};
},
};
},
[
path.resolve(__dirname, 'src/plugins/docusaurus-plugin-segment'),
{
writeKey: process.env.SEGMENT_TOKEN,
allowedInDev: false,
},
],
],
themeConfig:
/** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({
docs: {
versionPersistence: 'localStorage',
sidebar: {
hideable: true,
},
},
navbar: {
hideOnScroll: true,
title: 'Crawlee',
logo: {
src: 'img/crawlee-light.svg',
srcDark: 'img/crawlee-dark.svg',
},
items: [
{
type: 'doc',
docId: 'quick-start/quick-start',
label: 'Docs',
position: 'left',
},
{
type: 'doc',
docId: '/examples',
label: 'Examples',
position: 'left',
},
{
type: 'custom-api',
to: 'core',
label: 'API',
position: 'left',
activeBaseRegex: 'api/(?!.*/changelog)',
},
{
type: 'custom-api',
to: 'core/changelog',
label: 'Changelog',
position: 'left',
className: 'changelog',
activeBaseRegex: 'changelog',
},
{
to: 'blog',
label: 'Blog',
position: 'left',
},
],
},
colorMode: {
defaultMode: 'light',
disableSwitch: false,
respectPrefersColorScheme: true,
},
prism: {
defaultLanguage: 'typescript',
theme: require('prism-react-renderer').themes.github,
darkTheme: require('prism-react-renderer').themes.dracula,
additionalLanguages: ['docker', 'log', 'bash', 'diff', 'json'],
},
metadata: [
// eslint-disable-next-line max-len
{ name: 'description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },
// eslint-disable-next-line max-len
{ name: 'og:description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },
],
image: 'img/crawlee-js-og.png',
footer: {
links: [
{
title: 'Docs',
items: [
{
label: 'Guides',
to: 'js/docs/guides',
},
{
label: 'Examples',
to: 'js/docs/examples',
},
{
label: 'API reference',
to: 'js/api/core',
},
{
label: 'Changelog',
to: 'js/api/core/changelog',
},
],
},
{
title: 'Product',
items: [
{
label: 'Discord',
href: 'https://discord.com/invite/jyEM2PRvMU',
},
{
label: 'Stack Overflow',
href: 'https://stackoverflow.com/questions/tagged/crawlee',
},
{
label: 'Twitter',
href: 'https://twitter.com/apify',
},
{
label: 'YouTube',
href: 'https://www.youtube.com/apify',
},
],
},
{
title: 'More',
items: [
{
label: 'Apify platform',
href: 'https://apify.com',
},
{
label: 'Docusaurus',
href: 'https://docusaurus.io',
},
{
label: 'GitHub',
href: 'https://github.com/apify/crawlee',
},
],
},
],
},
algolia: {
appId: '5JC94MPMLY',
apiKey: '267679200b833c2ca1255ab276731869', // search only (public) API key
indexName: 'crawlee',
placeholder: 'Search documentation',
algoliaOptions: {
facetFilters: ['version:VERSION'],
},
translations: {
button: {
buttonText: 'Search documentation...',
},
},
},
}),
};