Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

4.3 KB
Raw Blame
module.exports = {
    docs: [
        'quick-start/quick-start',
        {
            type: 'category',
            label: 'Introduction',
            collapsed: false,
            link: {
                type: 'doc',
                id: 'introduction/introduction',
            },
            items: [
                'introduction/setting-up',
                'introduction/first-crawler',
                'introduction/adding-urls',
                'introduction/real-world-project',
                'introduction/crawling',
                'introduction/scraping',
                'introduction/saving-data',
                'introduction/refactoring',
                'introduction/deployment',
            ],
        },
        {
            type: 'category',
            label: 'Guides',
            link: {
                type: 'generated-index',
                title: 'Guides',
                slug: '/guides',
                keywords: ['guides'],
            },
            items: [
                'guides/request-storage',
                'guides/request-loaders',
                'guides/result-storage',
                'guides/http-clients',
                'guides/configuration',
                'guides/cheerio-crawler-guide',
                'guides/javascript-rendering',
                'guides/adaptive-crawling/adaptive-crawling',
                'guides/remote-browser',
                'guides/proxy-management',
                'guides/session-management',
                'guides/scaling-crawlers',
                'guides/multiple-crawlers',
                'guides/trace-and-monitor-crawlers',
                'guides/avoid-blocking',
                'guides/cookie-modals',
                'guides/impit-http-client/impit-http-client',
                'guides/send-request',
                'guides/typescript-project',
                'guides/public-api',
                'guides/docker-images',
                'guides/stagehand-crawler-guide',
                'guides/running-in-web-server/running-in-web-server',
                'guides/parallel-scraping/parallel-scraping-guide',
                'guides/custom-statistics',
                'guides/request-lifecycle/request-lifecycle',
                'guides/crawling-context/crawling-context',
                'guides/custom-http-client/custom-http-client',
                'guides/custom-logger/custom-logger'
            ],
        },
        {
            type: 'category',
            label: 'Deployment',
            link: {
                type: 'generated-index',
                title: 'Deployment guides',
                description: 'Here you can find guides on how to deploy your crawlers to various cloud providers.',
                slug: '/deployment',
            },
            items: [
                {
                    type: 'doc',
                    id: 'deployment/apify-platform',
                    label: 'Deploy on Apify',
                },
                {
                    type: 'category',
                    label: 'Deploy on AWS',
                    items: [
                        'deployment/aws-cheerio',
                        'deployment/aws-browsers',
                    ],
                },
                {
                    type: 'category',
                    label: 'Deploy to Google Cloud',
                    items: [
                        'deployment/gcp-cheerio',
                        'deployment/gcp-browsers',
                    ],
                },
            ],
        },
        {
            type: 'category',
            label: 'Examples',
            link: {
                type: 'generated-index',
                title: 'Examples',
                slug: '/examples',
                keywords: ['examples'],
            },
            items: [
                {
                    type: 'autogenerated',
                    dirName: 'examples',
                },
            ],
        },
        {
            type: 'category',
            label: 'Upgrading',
            link: {
                type: 'generated-index',
                title: 'Upgrading',
                slug: '/upgrading',
                keywords: ['upgrading'],
            },
            items: [
                {
                    type: 'autogenerated',
                    dirName: 'upgrading',
                },
            ],
        },
    ],
};