Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

16.2 KB
Raw Blame
import fs from 'node:fs';
import type { Server } from 'node:http';
import path from 'node:path';
import { setTimeout } from 'node:timers/promises';

import bodyParser from 'body-parser';
import type { Application } from 'express';
import express from 'express';
import iconv from 'iconv-lite';

import { entries } from './typedefs.js';

export const startExpressAppPromise = async (app: Application, port: number) => {
    return new Promise<Server>((resolve) => {
        const server = app.listen(port, () => resolve(server));
    });
};

export const responseSamples = {
    json: { foo: 'bar' },
    xml:
        '<?xml version="1.0" encoding="UTF-8"?>\n' +
        '<items>\n' +
        '<item>\n' +
        '    <url>https://apify.com</url>\n' +
        '    <title>Web Scraping, Data Extraction and Automation &#xb7; Apify</title>\n' +
        '</item>\n' +
        '</items>',
    complexXml: fs.readFileSync(path.join(import.meta.dirname, 'data/complex.xml'), 'utf-8'),
    image: fs.readFileSync(path.join(import.meta.dirname, 'data/apify.png')),
    html: `<!doctype html>
    <html>
    <head>
        <title>Example Domain</title>
        <meta charset="utf-8">
        <meta http-equiv="Content-type" content="text/html; charset=utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <style type="text/css">
        body {
            background-color: #f0f0f2;
            margin: 0;
            padding: 0;
            font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;

        }
        div {
            width: 600px;
            margin: 5em auto;
            padding: 2em;
            background-color: #fdfdff;
            border-radius: 0.5em;
            box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
        }
        a:link, a:visited {
            color: #38488f;
            text-decoration: none;
        }
        @media (max-width: 700px) {
            div {
                margin: 0 auto;
                width: auto;
            }
        }
        </style>
    </head>

    <body>
    <div>
        <h1>Example Domain</h1>
        <p>This domain is for use in illustrative examples in documents. You may use this
        domain in literature without prior coordination or asking for permission.</p>
        <p><a href="https://www.iana.org/domains/example">More information...</a></p>
    </div>
    </body>
    </html>`,
    resources: `
    <html><body>
            <link rel="stylesheet" type="text/css" href="/style.css">
            <img src="/image.png">
            <img src="/image.gif">
            <script src="/script.js" defer="defer"></script>
        </body></html>`,
    cacheable: {
        html: `
<!doctype html>
    <html>
    <head>
        <title>Cacheable example website</title>
        <meta charset="utf-8">
        <meta http-equiv="Content-type" content="text/html; charset=utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <link rel="stylesheet" href="cacheable/style.css">
        <script src="cacheable/script.js"></script>
    </body>
    </html>
`,
        css: `
body {
    background-color: #f0f0f2;
    margin: 0;
    padding: 0;
    font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
}
`,
        js: `
console.log('Hello world!');
`,
    },
    htmlWithOutsideRedirect: `
<html>
    <head>
        <title>Redirecting outside</title>
    </head>
    <body>
        <a href="/special/redirect-outside">click me</a>
    </body>
</html>`,
    // the real scripts live on https://<id>.<id>.<region>.token.awswaf.com and .captcha.awswaf.com,
    // the local paths keep the tests off the network
    awsWafChallenge: (bodyScript: string) => `
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title></title>
    <script type="text/javascript">
    window.awsWafCookieDomainList = ['dummypage.co'];
    window.gokuProps = { "key": "AQIDAHjcYu", "iv": "CgAGUjJldAAABiGQ", "context": "loYdKdld00dlbxeUxL9OCW1a9K3GDRIu" };
    </script>
    <script src="/special/fb423e1ef94f.6277d64d.us-east-1.token.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/challenge.js"></script>
</head>
<body>
    <div id="challenge-container"></div>
    ${bodyScript}
    <noscript>
        <h1>JavaScript is disabled</h1>
        In order to continue, we need to verify that you're not a robot.
    </noscript>
</body>
</html>`,
    awsWafCaptcha: `
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>Human Verification</title>
    <script type="text/javascript">
    window.awsWafCookieDomainList = ['dummypage.co'];
    window.gokuProps = { "key": "AQIDAHjcYu", "iv": "CgAGcDJmLAAABfB7", "context": "loYdKdld00dlbxeUxL9OCW1a9K3GDRIu" };
    </script>
    <script src="/special/fb423e1ef94f.6277d64d.us-east-1.token.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/challenge.js"></script>
    <script src="/special/fb423e1ef94f.6277d64d.us-east-1.captcha.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/captcha.js"></script>
</head>
<body>
    <div id="captcha-container"></div>
    <noscript>
        <h1>JavaScript is disabled</h1>
        In order to continue, you need to verify that you're not a robot by solving a CAPTCHA puzzle.
    </noscript>
</body>
</html>`,
    cloudflareBlocking: `
<!DOCTYPE html>
<head>
    <title>Just a moment...</title>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
    <meta name="robots" content="noindex,nofollow">
    <meta name="viewport" content="width=device-width,initial-scale=1">
    <link href="/cdn-cgi/styles/challenges.css" rel="stylesheet">
    <meta http-equiv="refresh" content="375">
    <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1?ray=837fec16d9534138"></script><script src="https://challenges.cloudflare.com/turnstile/v0/g/74bd6362/api.js?onload=AudPIu1&amp;render=explicit" async="" defer="" crossorigin="anonymous"></script>
</head>
<body class="no-js">
    <div class="main-wrapper" role="main">
        <div class="main-content">
            <h1 class="zone-name-title h1">dummypage.co</h1>
            <h2 id="challenge-running" class="h2">Checking if the site connection is secure</h2>
            <div id="challenge-stage" style="display: flex;">
                <div id="turnstile-wrapper" class="captcha-prompt spacer">
                    <div><iframe style="border: medium; overflow: hidden; width: 300px; height: 65px;" src="https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/g/turnstile/if/ov2/av0/rcv0/0/o9un1/0x4AAAAAAADnPIDROrmt1Wwj/dark/normal" allow="cross-origin-isolated; fullscreen" sandbox="allow-same-origin allow-scripts allow-popups" id="cf-chl-widget-o9un1" tabindex="0" title="Widget containing a Cloudflare security challenge"></iframe><input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-o9un1_response"></div>
                </div>
            </div>
            <div id="challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
                <div class="lds-ring">
                    <div></div>
                    <div></div>
                    <div></div>
                    <div></div>
                </div>
            </div>
            <div id="challenge-body-text" class="core-msg spacer">dummypage.co needs to review the security of your connection before proceeding.</div>
            <div id="challenge-success" style="display: none;">
                <div class="h2">
                    <span class="icon-wrapper">
                    </span>Connection is secure</div>
                <div class="core-msg spacer">Proceeding...</div>
            </div>
            <noscript>
                <div id="challenge-error-title">
                    <div class="h2">
                        <span class="icon-wrapper">
                            <div class="heading-icon warning-icon"></div>
                        </span>
                        <span id="challenge-error-text">Enable JavaScript and cookies to continue</span>
                    </div>
                </div>
            </noscript>
        </div>
    </div>
    <div class="footer" role="contentinfo">
        <div class="footer-inner">
            <div class="clearfix diagnostic-wrapper">
                <div class="ray-id">Ray ID: <code>837fec16d9534138</code></div>
            </div>
            <div class="text-center" id="footer-text">Performance &amp; security by <a rel="noopener noreferrer" href="https://www.cloudflare.com?utm_source=challenge&amp;utm_campaign=m" target="_blank">Cloudflare</a></div>
        </div>
    </div>
</body>
</html>`,
    outsideIframe: `
    <!DOCTYPE html>
    <html>
        <head>
            <title>Outside iframe title</title>
        </head>
        <body>
            <h1>Outside iframe</h1>
            <iframe src="./inside-iframe"></iframe>
        </body>
    </html>`,
    insideIframe: `
    <!DOCTYPE html>
    <html>
        <head>
            <title>In iframe</title>
        </head>
        <body>
            <h1>In iframe</h1>
            <p>Some content from inside of an iframe.</p>
        </body>
    </html>`,
    shadowRoots: `
    <html>
    <body>
        <div id="open-container">
            <template shadowrootmode="open">
                <p>[GOOD] This is inside the OPEN shadow DOM.</p>
                <div>
                    <template shadowrootmode="open">
                        <p>[GOOD] This is inside of inside of the OPEN shadow DOM.</p>
                    </template>
                </div>
            </template>
        </div>
        <div id="closed-container">
            <template shadowrootmode="closed">
                <p>[BAD] This is inside the CLOSED shadow DOM.</p>
                <div>
                    <template shadowrootmode="open">
                        <p>[BAD] This is inside of inside of the CLOSED shadow DOM.</p>
                    </template>
                </div>
            </template>
        </div>
    </body>
    </html>`,
};

export async function runExampleComServer(): Promise<[Server, number]> {
    const app = express();

    app.use(
        bodyParser.urlencoded({
            extended: true,
        }),
    );
    app.use(bodyParser.json());

    const special = express.Router();
    const cacheable = express.Router();

    // "special" pages with debugging info and responses for use in tests
    (() => {
        special.get('/getRawHeaders', (req, res) => {
            res.send(JSON.stringify(req.rawHeaders));
        });

        special.all('/getDebug', (req, res) => {
            res.json({
                headers: req.headers,
                method: req.method,
                bodyLength: +(req.headers['content-length'] ?? 0),
            });
        });

        special.post('/mock', (req, res) => {
            const { headers, statusCode, error = false, body } = req.body;

            if (error) {
                throw new Error(error);
            }

            entries(headers as Record<string, string>).forEach(([key, value]) => res.setHeader(key, value));

            res.status(statusCode).send(body);
        });

        special.get('/headers', (req, res) => {
            res.status(200).json({ headers: req.headers });
        });

        special.get('/invalidContentType', (_req, res) => {
            res.send({ some: 'json' });
        });

        special.post('/jsonError', (_req, res) => {
            res.status(500).json({ message: 'CUSTOM_ERROR' });
        });

        special.get('/mirror', (_req, res) => {
            res.send('<html><head><title>Title</title></head><body>DATA</body></html>');
        });

        special.get('/html-type', (_req, res) => {
            res.send(responseSamples.html);
        });

        special.get('/json-type', (_req, res) => {
            res.json(responseSamples.json);
        });
        special.get('/xml-type', (_req, res) => {
            res.type('application/xml');
            res.send(responseSamples.xml);
        });
        special.get('/complex-xml', (_req, res) => {
            res.type('application/xml');
            res.send(responseSamples.complexXml);
        });
        special.get('/image-type', (_req, res) => {
            res.type('image/png');
            res.send(responseSamples.image);
        });

        special.get('/timeout', async (_req, res) => {
            await setTimeout(32000);
            res.type('html').send('<div>TEST</div>');
        });
        special.get('/resources', async (_req, res) => {
            res.type('html').send(responseSamples.resources);
        });

        special.get('/redirect', (_req, res) => {
            res.type('html').send(responseSamples.htmlWithOutsideRedirect);
        });

        special.get('/redirect-outside', (req, res) => {
            res.redirect('https://example.com');
        });

        special.get('/cloudflareBlocking', async (_req, res) => {
            res.type('html').status(403).send(responseSamples.cloudflareBlocking);
        });

        special.get('/awsWafChallengePassed', async (_req, res) => {
            // a passed challenge reloads the page, which then gets the real content
            const pass = `<script>setTimeout(() => window.location.replace('/special/html-type'), 500);</script>`;
            res.type('html').status(202).send(responseSamples.awsWafChallenge(pass));
        });

        special.get('/awsWafChallengeFailed', async (_req, res) => {
            // a failed challenge reloads the page as well, and gets the CAPTCHA instead
            const fail = `<script>setTimeout(() => window.location.replace('/special/awsWafCaptcha'), 500);</script>`;
            res.type('html').status(202).send(responseSamples.awsWafChallenge(fail));
        });

        special.get('/awsWafCaptcha', async (_req, res) => {
            res.type('html').status(405).send(responseSamples.awsWafCaptcha);
        });

        special.get('/outside-iframe', (_req, res) => {
            res.type('html').send(responseSamples.outsideIframe);
        });

        special.get('/outside-iframe-csp', (_req, res) => {
            res.setHeader('Content-Security-Policy', "require-trusted-types-for 'script'");
            res.type('html').send(responseSamples.outsideIframe);
        });

        special.get('/inside-iframe', (_req, res) => {
            res.type('html').send(responseSamples.insideIframe);
        });

        special.get('/shadow-root', (_req, res) => {
            res.type('html').send(responseSamples.shadowRoots);
        });

        special.get('/html-entities', (_req, res) => {
            res.type('html').send('&quot;&lt;&gt;"<>');
        });

        special.get('/meta-charset', (_req, res) => {
            const text = 'Žluťoučký kůň';
            const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
            res.setHeader('content-type', 'text/html');
            res.end(iconv.encode(html, 'windows-1250'));
        });

        special.get('/set-cookie', (req, res) => {
            const cookieName = (req.query.name as string) || 'testCookie';
            const cookieValue = (req.query.value as string) || 'testValue';
            res.setHeader('set-cookie', `${cookieName}=${cookieValue}; Path=/`);
            res.type('html').send('<html><body>Cookie set</body></html>');
        });

        special.get('/get-cookies', (req, res) => {
            res.json({ cookies: req.headers.cookie || '' });
        });
    })();

    // "cacheable" site with one page, scripts and stylesheets
    (() => {
        cacheable.get('/', (req, res) => {
            res.send(responseSamples.cacheable.html);
        });
        cacheable.get('/style.css', (req, res) => {
            res.type('text/css').send(responseSamples.cacheable.css);
        });
        cacheable.get('/script.js', (req, res) => {
            res.type('application/javascript').send(responseSamples.cacheable.js);
        });
    })();

    app.use('/special', special);
    app.use('/cacheable', cacheable);

    app.get('{*splat}', async (req, res) => {
        await setTimeout(50);
        res.send(responseSamples.html);
    });

    const server = await startExpressAppPromise(app, 0);
    const { port } = server.address() as any;

    return [server, port];
}