Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

28.3 KB
Raw Blame
import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import { promisify } from 'node:util';

import type { CommonLibrary, UnwrapPromise } from '@crawlee/browser-pool';
import {
    LaunchContext,
    PlaywrightBrowser,
    PlaywrightController,
    PlaywrightPlugin,
    PuppeteerController,
    PuppeteerPlugin,
} from '@crawlee/browser-pool';
import playwright from 'playwright';
import type { Server as ProxyChainServer } from 'proxy-chain';
import type { Browser } from 'puppeteer';
import puppeteer from 'puppeteer';
import { runExampleComServer } from '../../shared/_helper.js';

import { createProxyServer } from './create-proxy-server.js';

// Chromium goes through the tests faster than Firefox and the results are more useful than with
// webkit. CI and `pnpm test:full` set CRAWLEE_TEST_BROWSERS to the full list.
const requestedBrowsers = (process.env.CRAWLEE_TEST_BROWSERS ?? 'chromium').split(',');
const testedBrowsers = (['chromium', 'firefox', 'webkit'] as const).filter((name) => requestedBrowsers.includes(name));

// Firefox browser launch is significantly slower than Chromium/WebKit (~12s vs <1s).
// Under CPU load from parallel tests, it can exceed 2 minutes. Use 5 minute timeout.
vitest.setConfig({ testTimeout: 300_000 });

let port: number;
let server: Server;
let serverAddress = 'http://localhost:';

beforeAll(async () => {
    [server, port] = await runExampleComServer();
    serverAddress += port;
});

afterAll(() => {
    server.close();
});

const runPluginTest = <
    P extends typeof PlaywrightPlugin | typeof PuppeteerPlugin,
    C extends typeof PuppeteerController | typeof PlaywrightController,
    L extends CommonLibrary,
>(
    Plugin: P,
    Controller: C,
    library: L,
) => {
    describe(`${library.constructor.name} - ${'name' in library ? library.name!() : ''} general`, () => {
        test.concurrent('should launch browser', async () => {
            const plugin = new Plugin(library as never);
            const browser = await plugin.launch();
            try {
                expect(typeof browser.newPage).toBe('function');
                expect(typeof browser.close).toBe('function');
            } finally {
                await browser.close();
            }
        });

        test.concurrent('should create launch context', () => {
            const plugin = new Plugin(library as never);
            const id = 'abc';
            const launchOptions = { foo: 'bar' };
            const proxyUrl = 'http://proxy.com/';
            const context = plugin.createLaunchContext({
                id,
                // @ts-expect-error Testing options
                launchOptions,
            });

            expect(context).toBeInstanceOf(LaunchContext);

            context.proxyUrl = proxyUrl;
            context.extend({
                one: 1,
            });

            const desiredObject = {
                id,
                launchOptions,
                browserPlugin: plugin,
                proxyUrl: proxyUrl.slice(0, -1),
                one: 1,
                useIncognitoPages: false,
            };

            // expect(context).toMatchObject(desiredObject)
            // Switch to this after the issue with `TypeError: prop.startsWith is not a function` is solved.

            expect(context.id).toEqual(desiredObject.id);
            expect(context.launchOptions).toEqual(desiredObject.launchOptions);
            expect(context.browserPlugin).toEqual(desiredObject.browserPlugin);
            expect(context.proxyUrl).toEqual(desiredObject.proxyUrl);
            expect(context.one).toEqual(desiredObject.one);
            expect(context.useIncognitoPages).toEqual(desiredObject.useIncognitoPages);
        });

        test.concurrent('should get default launchContext values from plugin options', async () => {
            const proxyUrl = 'http://[email protected]:8080/';

            const plugin = new Plugin(library as never, {
                proxyUrl,
                userDataDir: 'test',
                useIncognitoPages: true,
            });

            const context = plugin.createLaunchContext();

            expect(context.proxyUrl).toEqual(proxyUrl.slice(0, -1));
            expect(context.useIncognitoPages).toBeTruthy();
            expect(context.userDataDir).toEqual('test');
        });

        test.concurrent('should keep the port of a socks proxyUrl from plugin options', () => {
            const plugin = new Plugin(library as never, { proxyUrl: 'socks5://127.0.0.1:1080' });
            const context = plugin.createLaunchContext();

            expect(context.proxyUrl).toEqual('socks5://127.0.0.1:1080');
        });

        test.concurrent('should allow unsetting launchContext proxyUrl', () => {
            const plugin = new Plugin(library as never);
            const context = plugin.createLaunchContext();

            context.proxyUrl = 'http://proxy.com/';
            expect(context.proxyUrl).toEqual('http://proxy.com');

            context.proxyUrl = undefined;
            expect(context.proxyUrl).toBeUndefined();
        });

        test.concurrent('should create browser controller', () => {
            const plugin = new Plugin(library as never);
            const browserController = plugin.createController();
            expect(browserController).toBeInstanceOf(Controller);
        });

        test.concurrent('should work with cookies', async () => {
            const plugin = new Plugin(library as never);
            const browserController = plugin.createController();
            const context = plugin.createLaunchContext();

            const browser = await plugin.launch(context as never);

            try {
                browserController.assignBrowser(browser as never, context as never);
                browserController.activate();

                const page = await browserController.newPage();
                await browserController.setCookies(page as never, [
                    { name: 'TEST', value: 'TESTER-COOKIE', url: serverAddress },
                ]);
                await page.goto(serverAddress, { waitUntil: 'domcontentloaded' });

                const cookies = await browserController.getCookies(page as never);
                expect(cookies[0].name).toBe('TEST');
                expect(cookies[0].value).toBe('TESTER-COOKIE');
            } finally {
                await browser.close();
            }
        });

        test.concurrent('newPage options cannot be used with persistent context', async () => {
            const plugin = new Plugin(library as never);
            const browserController = plugin.createController();

            const context = plugin.createLaunchContext({
                useIncognitoPages: false,
            });

            const browser = await plugin.launch(context as never);
            try {
                browserController.assignBrowser(browser as never, context as never);
                browserController.activate();

                try {
                    const page = await browserController.newPage({});
                    await page.close();

                    expect(false).toBe(true);
                } catch (error: any) {
                    expect(error.message).toBe(
                        'A new page can be created with provided context only when using incognito pages.',
                    );
                }
            } finally {
                await browser.close();
            }
        });
    });
};

describe('Plugins', () => {
    let target: http.Server;
    let unprotectedProxy: ProxyChainServer;
    let protectedProxy: ProxyChainServer;

    beforeAll(async () => {
        target = http.createServer((request, response) => {
            response.end(request.socket.remoteAddress);
        });
        await promisify(target.listen.bind(target) as any)(0, '127.0.0.1');

        unprotectedProxy = createProxyServer('127.0.0.2', '', '');
        await unprotectedProxy.listen();

        protectedProxy = createProxyServer('127.0.0.3', 'foo', 'bar');
        await protectedProxy.listen();
    });

    afterAll(async () => {
        await promisify(target.close.bind(target))();

        await unprotectedProxy.close(false);
        await protectedProxy.close(false);
    });

    describe('Puppeteer specifics', () => {
        test.concurrent('should work with non authenticated proxyUrl', async () => {
            const proxyUrl = `http://127.0.0.2:${unprotectedProxy.port}`;
            const plugin = new PuppeteerPlugin(puppeteer);

            const context = plugin.createLaunchContext({
                proxyUrl,
                launchOptions: {
                    args: [
                        // Exclude loopback interface from proxy bypass list,
                        // so the request to localhost goes through proxy.
                        // This way there's no need for a 3rd party server.
                        '--proxy-bypass-list=<-loopback>',
                    ],
                },
            });

            const browser = await plugin.launch(context);

            try {
                const page = await browser.newPage();
                const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);

                const text = await response!.text();

                expect(text).toBe('127.0.0.2');

                await page.close();
            } finally {
                await browser.close();
            }
        });

        test.concurrent('should work with authenticated proxyUrl', async () => {
            const proxyUrl = `http://foo:[email protected]:${protectedProxy.port}`;
            const plugin = new PuppeteerPlugin(puppeteer);

            const context = plugin.createLaunchContext({
                proxyUrl,
                launchOptions: {
                    args: [
                        // Exclude loopback interface from proxy bypass list,
                        // so the request to localhost goes through proxy.
                        // This way there's no need for a 3rd party server.
                        '--proxy-bypass-list=<-loopback>',
                    ],
                },
            });

            const browser = await plugin.launch(context);

            try {
                const page = await browser.newPage();
                const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);

                const text = await response!.text();

                expect(text).toBe('127.0.0.3');

                await page.close();
            } finally {
                await browser.close();
            }
        });

        test.concurrent('should use persistent context by default', async () => {
            const plugin = new PuppeteerPlugin(puppeteer);
            const browserController = plugin.createController();

            const launchContext = plugin.createLaunchContext();

            const browser = await plugin.launch(launchContext);
            try {
                browserController.assignBrowser(browser, launchContext);
                browserController.activate();

                const page = await browserController.newPage();
                const browserContext = page.browserContext();

                expect(browserContext.id).toBeFalsy();
            } finally {
                await browser.close();
            }
        });

        test.concurrent('should use incognito pages by option', async () => {
            const plugin = new PuppeteerPlugin(puppeteer);
            const browserController = plugin.createController();

            const launchContext = plugin.createLaunchContext({ useIncognitoPages: true });

            const browser = await plugin.launch(launchContext);
            try {
                browserController.assignBrowser(browser, launchContext);
                browserController.activate();

                const page = await browserController.newPage();
                const browserContext = page.browserContext();

                expect(browserContext.id).toBeTruthy();
            } finally {
                await browser.close();
            }
        });

        test.concurrent('should allow calling functions on the proxied browser', async () => {
            const plugin = new PuppeteerPlugin(puppeteer);
            const launchContext = plugin.createLaunchContext();
            const browser = await plugin.launch(launchContext);

            try {
                await expect(browser.pages()).resolves.toBeDefined();
            } finally {
                await browser.close();
            }
        });

        test.concurrent('should pass launch options to browser', async () => {
            const plugin = new PuppeteerPlugin(puppeteer);

            const userAgent = 'HelloWorld';

            const launchOptions = {
                args: [`--user-agent=${userAgent}`],
            };

            const launchContext = plugin.createLaunchContext({ launchOptions });
            const browser = await plugin.launch(launchContext);

            try {
                expect(await browser.userAgent()).toBe(userAgent);
            } finally {
                await browser.close();
            }
        });

        test.concurrent('proxyUsername and proxyPassword as newPage options', async () => {
            const plugin = new PuppeteerPlugin(puppeteer);
            const browserController = new PuppeteerController(plugin);

            const launchContext = plugin.createLaunchContext({
                useIncognitoPages: true,
            });

            const browser = await plugin.launch(launchContext);
            try {
                browserController.assignBrowser(browser, launchContext);
                browserController.activate();

                const page = await browserController.newPage({
                    proxyServer: `http://127.0.0.3:${protectedProxy.port}`,
                    proxyUsername: 'foo',
                    proxyPassword: 'bar',
                    proxyBypassList: ['<-loopback>'],
                });

                const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
                const text = await response!.text();

                expect(text).toBe('127.0.0.3');

                await page.close();
            } finally {
                await browser.close();
            }
        });
    });

    runPluginTest(PuppeteerPlugin, PuppeteerController, puppeteer);

    describe('normalizeProxyOptions', () => {
        test.each([
            ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],
            ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],
        ])('PlaywrightController should handle %s', (proxyUrl, expectedServer) => {
            const plugin = new PlaywrightPlugin(playwright.chromium);
            const browserController = new PlaywrightController(plugin);

            const result = browserController.normalizeProxyOptions(proxyUrl, {});

            expect(result).toMatchObject({
                proxy: { server: expectedServer, username: 'user', password: 'pass' },
            });
        });

        test.each([
            ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],
            ['socks://user:[email protected]:1080', 'socks://proxy.example.com:1080'],
            ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],
        ])(
            'PlaywrightPlugin.addProxyToLaunchOptions should set correct server for %s',
            async (proxyUrl, expectedServer) => {
                const plugin = new PlaywrightPlugin(playwright.chromium);
                const context = plugin.createLaunchContext({ proxyUrl });
                await (plugin as any).addProxyToLaunchOptions(context);
                expect(context.launchOptions!.proxy!.server).toEqual(expectedServer);
            },
        );

        test.each([
            ['socks5://user:[email protected]:1080', 'socks5://proxy.example.com:1080'],
            ['http://user:[email protected]:8080', 'http://proxy.example.com:8080'],
        ])('PuppeteerController should handle %s', (proxyUrl, expectedServer) => {
            const plugin = new PuppeteerPlugin(puppeteer);
            const browserController = new PuppeteerController(plugin);

            const result = browserController.normalizeProxyOptions(proxyUrl, {});

            expect(result).toMatchObject({
                proxyServer: expectedServer,
                proxyUsername: 'user',
                proxyPassword: 'pass',
            });
        });
    });

    describe('Playwright specifics', () => {
        let browser: playwright.Browser;

        afterEach(async () => {
            await browser.close();
        });

        describe.each(testedBrowsers)('with %s', (browserName) => {
            test('should work with non authenticated proxyUrl', async () => {
                const proxyUrl = `http://127.0.0.2:${unprotectedProxy.port}`;
                const plugin = new PlaywrightPlugin(playwright[browserName]);

                const launchOptions =
                    browserName === 'chromium'
                        ? {
                              args: [
                                  // Exclude loopback interface from proxy bypass list,
                                  // so the request to localhost goes through proxy.
                                  // This way there's no need for a 3rd party server.
                                  '--proxy-bypass-list=<-loopback>',
                              ],
                          }
                        : undefined;

                const context = plugin.createLaunchContext({
                    proxyUrl,
                    launchOptions,
                });

                browser = await plugin.launch(context);
                expect(context.launchOptions!.proxy!.server).toEqual(proxyUrl);

                const page = await browser.newPage();
                const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
                const text = await response!.text();

                expect(text).toBe('127.0.0.2');

                await page.close();
            });

            test('should work with authenticated proxyUrl', async () => {
                const proxyUrl = `http://foo:[email protected]:${protectedProxy.port}`;
                const plugin = new PlaywrightPlugin(playwright[browserName]);

                const launchOptions =
                    browserName === 'chromium'
                        ? {
                              args: [
                                  // Exclude loopback interface from proxy bypass list,
                                  // so the request to localhost goes through proxy.
                                  // This way there's no need for a 3rd party server.
                                  '--proxy-bypass-list=<-loopback>',
                              ],
                          }
                        : undefined;

                const context = plugin.createLaunchContext({
                    proxyUrl,
                    launchOptions,
                });

                browser = await plugin.launch(context);

                const page = await browser.newPage();
                const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
                const text = await response!.text();

                expect(text).toBe('127.0.0.3');

                await page.close();
            });

            test('proxy as newPage option', async () => {
                const plugin = new PlaywrightPlugin(playwright.chromium);
                const browserController = new PlaywrightController(plugin);

                const launchContext = plugin.createLaunchContext({
                    useIncognitoPages: true,
                });

                browser = await plugin.launch(launchContext);
                browserController.assignBrowser(browser, launchContext);
                browserController.activate();

                const page = await browserController.newPage({
                    proxy: {
                        server: `http://127.0.0.3:${protectedProxy.port}`,
                        username: 'foo',
                        password: 'bar',
                        bypass: '<-loopback>',
                    },
                });

                const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`);
                const text = await response!.text();

                expect(text).toBe('127.0.0.3');

                await page.close();
            });

            test('should use incognito context by option', async () => {
                const plugin = new PlaywrightPlugin(playwright[browserName]);
                const browserController = plugin.createController();

                const launchContext = plugin.createLaunchContext({ useIncognitoPages: true });

                browser = await plugin.launch(launchContext);
                browserController.assignBrowser(browser, launchContext);
                browserController.activate();

                const page = await browserController.newPage();
                const browserContext = page.context();
                await browserController.newPage();

                expect(browserContext.pages()).toHaveLength(1);
            });

            test('should use persistent context by default', async () => {
                const plugin = new PlaywrightPlugin(playwright[browserName]);
                const browserController = plugin.createController();

                const launchContext = plugin.createLaunchContext();

                browser = await plugin.launch(launchContext);
                browserController.assignBrowser(browser, launchContext);
                browserController.activate();

                const page = await browserController.newPage();
                const context = page.context();
                await browserController.newPage();

                expect(context.pages()).toHaveLength(3); // 3 pages because of the about:blank.
            });

            test('should pass launch options to browser', async () => {
                const plugin = new PlaywrightPlugin(playwright[browserName]);

                let ran = false;

                const launchOptions = {
                    logger: {
                        isEnabled: () => {
                            ran = true;
                            return false;
                        },
                        log: () => {},
                    },
                };

                const launchContext = plugin.createLaunchContext({ launchOptions });
                browser = await plugin.launch(launchContext);

                expect(ran).toBe(true);
            });

            describe('PlaywrightBrowser', () => {
                test('should create new page', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext();
                    browser = await plugin.launch(launchContext);
                    const page = await browser.newPage();

                    expect(typeof page.close).toBe('function');
                    expect(typeof page.evaluate).toBe('function');
                });

                test('should emit disconnected event on close', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext();
                    browser = await plugin.launch(launchContext);
                    let called = false;

                    browser.on('disconnected', () => {
                        called = true;
                    });

                    await browser.close();

                    expect(called).toBe(true);
                });

                test('should be used only with incognito pages context', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext({ useIncognitoPages: false });
                    browser = await plugin.launch(launchContext);
                    expect(browser).toBeInstanceOf(PlaywrightBrowser);

                    await browser.close();

                    const launchContext2 = plugin.createLaunchContext({ useIncognitoPages: true });
                    browser = await plugin.launch(launchContext2);
                    expect(browser).not.toBeInstanceOf(PlaywrightBrowser);
                });

                test('should return correct version', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext({ useIncognitoPages: false });
                    browser = await plugin.launch(launchContext);
                    const version1 = browser.version();

                    await browser.close();

                    const launchContext2 = plugin.createLaunchContext({ useIncognitoPages: true });
                    browser = await plugin.launch(launchContext2);
                    expect(version1).toEqual(browser.version());
                });

                test('should return browser type', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext({ useIncognitoPages: false });
                    browser = await plugin.launch(launchContext);
                    expect(browser.browserType()).toBe(playwright[browserName]);

                    await browser.close();

                    const launchContext2 = plugin.createLaunchContext({ useIncognitoPages: true });
                    browser = await plugin.launch(launchContext2);
                    expect(browser.browserType()).toBe(playwright[browserName]);
                });

                test('should return all contexts', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext();
                    browser = await plugin.launch(launchContext);
                    const contexts = browser.contexts();
                    expect(contexts).toHaveLength(1);

                    // the returned context is the one new pages are created in
                    const page = await browser.newPage();
                    expect(page.context()).toBe(contexts[0]);
                });

                test('should return correct connected status', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);

                    const launchContext = plugin.createLaunchContext();
                    browser = await plugin.launch(launchContext);
                    expect(browser.isConnected()).toBe(true);

                    await browser.close();

                    expect(browser.isConnected()).toBe(false);
                });

                test('should throw on newContext call', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);
                    const launchContext = plugin.createLaunchContext();
                    browser = await plugin.launch(launchContext);

                    await expect(browser.newContext()).rejects.toThrow(
                        'Function `newContext()` is not available in incognito mode',
                    );
                });

                test('should have same public interface as playwright browserType', async () => {
                    const plugin = new PlaywrightPlugin(playwright[browserName]);
                    const originalFunctionNames = [
                        'close',
                        'contexts',
                        'isConnected',
                        'newContext',
                        'newPage',
                        'version',
                    ] as const;
                    const launchContext = plugin.createLaunchContext({ useIncognitoPages: true });
                    browser = await plugin.launch(launchContext);

                    for (const originalFunctionName of originalFunctionNames) {
                        expect(typeof browser[originalFunctionName]).toBe('function');
                    }

                    expect.hasAssertions();
                });
            });
        });
    });

    for (const browserName of testedBrowsers) {
        runPluginTest(PlaywrightPlugin, PlaywrightController, playwright[browserName]);
    }
});