Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

17.6 KB
Raw Blame
import type { Server } from 'node:http';
import path from 'node:path';

import { MemoryStorageBackend, serviceLocator } from '@crawlee/core';
import { KeyValueStore, launchPuppeteer, puppeteerUtils, Request } from '@crawlee/puppeteer';
import type { Browser, Page } from 'puppeteer';
import { runExampleComServer } from '../shared/_helper.js';

import log from '@apify/log';

const launchContext = { launchOptions: { headless: true } };

let port: number;
let server: Server;
let serverAddress = 'http://localhost:';

beforeAll(async () => {
    [server, port] = await runExampleComServer();
    serverAddress += port;
});

afterAll(() => {
    server.close();
});

describe('puppeteerUtils', () => {
    let ll: number;

    beforeAll(async () => {
        ll = log.getLevel();
        log.setLevel(log.LEVELS.ERROR);
    });

    beforeEach(async () => {
        serviceLocator.setStorageBackend(new MemoryStorageBackend());
    });

    afterAll(async () => {
        log.setLevel(ll);
    });

    describe('with %s', () => {
        test('injectFile()', async () => {
            const browser2 = await launchPuppeteer(launchContext);
            const survive = async (browser: Browser) => {
                // Survive navigations
                const page = await browser.newPage();
                // @ts-expect-error
                let result = await page.evaluate(() => window.injectedVariable === 42);
                expect(result).toBe(false);
                await puppeteerUtils.injectFile(
                    page,
                    path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),
                    {
                        surviveNavigations: true,
                    },
                );
                // @ts-expect-error
                result = await page.evaluate(() => window.injectedVariable);
                expect(result).toBe(42);
                await page.goto('about:chrome');
                // @ts-expect-error
                result = await page.evaluate(() => window.injectedVariable);
                expect(result).toBe(42);
                await page.goto(serverAddress);
                // @ts-expect-error
                result = await page.evaluate(() => window.injectedVariable);
                expect(result).toBe(42);
            };
            const remove = async (browser: Browser) => {
                // Remove with navigations
                const page = await browser.newPage();
                // @ts-expect-error
                let result = await page.evaluate(() => window.injectedVariable === 42);
                expect(result).toBe(false);
                await page.goto('about:chrome');
                // @ts-expect-error
                result = await page.evaluate(() => window.injectedVariable === 42);
                expect(result).toBe(false);
                await puppeteerUtils.injectFile(
                    page,
                    path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),
                );
                // @ts-expect-error
                result = await page.evaluate(() => window.injectedVariable);
                expect(result).toBe(42);
                await page.goto(serverAddress);
                // @ts-expect-error
                result = await page.evaluate(() => window.injectedVariable === 42);
                expect(result).toBe(false);
            };
            try {
                await Promise.all([survive(browser2), remove(browser2)]);
            } finally {
                await browser2.close();
            }
        });

        test('injectJQuery()', async () => {
            const browser = await launchPuppeteer(launchContext);

            try {
                const page = await browser.newPage();
                await page.goto('about:blank');

                // NOTE: Chrome already defines window.$ as alias to document.querySelector(),
                // (https://developers.google.com/web/tools/chrome-devtools/console/command-line-reference#queryselector)
                const result1 = await page.evaluate(() => {
                    return {
                        // @ts-expect-error
                        isDefined: window.jQuery !== undefined,
                    };
                });
                expect(result1).toEqual({
                    isDefined: false,
                });

                await puppeteerUtils.injectJQuery(page);
                const result2 = await page.evaluate(() => {
                    /* global $ */
                    return {
                        // @ts-expect-error
                        isDefined: window.jQuery === window.$,
                        // @ts-expect-error
                        text: $('h1').text(),
                    };
                });
                expect(result2).toEqual({
                    isDefined: true,
                    text: '',
                });

                await page.reload();

                const result3 = await page.evaluate(() => {
                    return {
                        // @ts-expect-error
                        isDefined: window.jQuery === window.$,
                        // @ts-expect-error
                        text: $('h1').text(),
                    };
                });
                expect(result3).toEqual({
                    isDefined: true,
                    text: '',
                });
            } finally {
                await browser.close();
            }
        });

        test('parseWithCheerio() works', async () => {
            const browser = await launchPuppeteer(launchContext);

            try {
                const page = await browser.newPage();
                await page.goto(serverAddress);

                const $ = await puppeteerUtils.parseWithCheerio(page);

                const title = $('h1').text().trim();
                expect(title).toBe('Example Domain');
            } finally {
                await browser.close();
            }
        });

        test('parseWithCheerio() iframe expansion works', async () => {
            const browser = await launchPuppeteer(launchContext);

            try {
                const page = await browser.newPage();
                await page.goto(new URL('/special/outside-iframe', serverAddress).toString());

                const $ = await puppeteerUtils.parseWithCheerio(page);

                const headings = $('h1')
                    .map((_, el) => $(el).text())
                    .get();

                const titles = $('title')
                    .map((_, el) => $(el).text())
                    .get();

                expect(titles).toEqual(['Outside iframe title']);
                expect(headings).toEqual(['Outside iframe', 'In iframe']);
            } finally {
                await browser.close();
            }
        });

        describe('parseWithCheerio() shadow root expansion works', () => {
            let browser: Browser;
            beforeAll(async () => {
                browser = await launchPuppeteer(launchContext);
            });
            afterAll(async () => {
                await browser.close();
            });

            // TODO verify with others how this behaves
            test('no expansion with ignoreShadowRoots: true', async () => {
                const page = await browser.newPage();
                await page.goto(`${serverAddress}/special/shadow-root`);
                const result = await puppeteerUtils.parseWithCheerio(page, true);
                const text = result('body').text().trim();

                // this is failing on macos
                if (process.platform !== 'darwin') {
                    expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(0);
                    expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0);
                }
            });

            test('expansion works', async () => {
                const page = await browser.newPage();
                await page.goto(`${serverAddress}/special/shadow-root`);
                const result = await puppeteerUtils.parseWithCheerio(page);
                const text = result('body').text().trim();

                // this is failing on macos
                if (process.platform !== 'darwin') {
                    expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(2);
                    expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0);
                }
            });
        });

        describe('blockRequests()', () => {
            let browser: Browser = null as any;
            beforeAll(async () => {
                browser = await launchPuppeteer(launchContext);
            });
            afterAll(async () => {
                await browser.close();
            });

            test('works with default values', async () => {
                const loadedUrls: string[] = [];

                const page = await browser.newPage();
                await puppeteerUtils.blockRequests(page);
                page.on('response', (response) => loadedUrls.push(response.url()));
                await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'load' });
                expect(loadedUrls).toEqual([`${serverAddress}/special/resources`, `${serverAddress}/script.js`]);
            });

            test('works with overridden values', async () => {
                const loadedUrls: string[] = [];

                const page = await browser.newPage();
                await puppeteerUtils.blockRequests(page, {
                    urlPatterns: ['.css'],
                });
                page.on('response', (response) => loadedUrls.push(response.url()));
                await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });

                expect(loadedUrls).toEqual(
                    expect.arrayContaining([
                        `${serverAddress}/image.png`,
                        `${serverAddress}/script.js`,
                        `${serverAddress}/image.gif`,
                    ]),
                );
            });
        });

        test('compileScript() works', async () => {
            const { compileScript } = puppeteerUtils;
            const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';
            const scriptStringBad = 'for const while';
            const script = compileScript(scriptStringGood);

            expect(typeof script).toBe('function');
            expect(script.toString()).toEqual(`async ({ page, request }) => {${scriptStringGood}}`);

            try {
                compileScript(scriptStringBad);
                throw new Error('Should fail.');
            } catch (err) {
                // TODO figure out why the err.message comes out empty in the logs.
                expect((err as Error).message).toMatch(/Unexpected token '?const'?/);
            }
            const browser = await launchPuppeteer(launchContext);
            try {
                const page = await browser.newPage();
                const content = await script({ page } as any);
                expect(typeof content).toBe('string');
                expect(content).toBe('<html><head></head><body></body></html>');
            } finally {
                await browser.close();
            }
        });

        test('gotoExtended() works', async () => {
            const browser = await launchPuppeteer(launchContext);

            try {
                const page = await browser.newPage();
                const request = new Request({
                    url: `${serverAddress}/special/getDebug`,
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json; charset=utf-8',
                        'User-Agent': 'Demo UA',
                    },
                    payload: '{ "foo": "bar" }',
                });

                const response = await puppeteerUtils.gotoExtended(page, request, { waitUntil: 'networkidle' });

                const { method, headers, bodyLength } = JSON.parse(await response!.text());
                expect(method).toBe('POST');
                expect(bodyLength).toBe(16);
                expect(headers['content-type']).toBe('application/json; charset=utf-8');
                expect(headers['user-agent']).toBe('Demo UA');
            } finally {
                await browser.close();
            }
        });

        test('gotoExtended() applies headers natively for GET requests without a payload', async () => {
            const browser = await launchPuppeteer(launchContext);

            try {
                const page = await browser.newPage();
                const interceptionSpy = vitest.spyOn(page, 'setRequestInterception');
                const deprecatedSpy = vitest.spyOn(log, 'deprecated');

                const request = new Request({
                    url: `${serverAddress}/special/getDebug`,
                    headers: {
                        'User-Agent': 'Demo UA',
                    },
                });

                const response = await puppeteerUtils.gotoExtended(page, request, { waitUntil: 'networkidle' });

                const { method, headers } = JSON.parse(await response!.text());
                expect(method).toBe('GET');
                expect(headers['user-agent']).toBe('Demo UA');
                expect(interceptionSpy).not.toHaveBeenCalled();
                expect(deprecatedSpy).not.toHaveBeenCalled();
            } finally {
                await browser.close();
            }
        });

        describe('infiniteScroll()', () => {
            function isAtBottom() {
                return window.innerHeight + window.pageYOffset >= document.body.offsetHeight;
            }

            let browser: Browser;
            beforeAll(async () => {
                browser = await launchPuppeteer({ launchOptions: { headless: true } });
            });
            afterAll(async () => {
                await browser.close();
            });

            let page: Page;
            beforeEach(async () => {
                page = await browser.newPage();
                let count = 0;
                const content = Array(1000)
                    .fill(null)
                    .map(() => {
                        return `<div style="border: 1px solid black">Div number: ${count++}</div>`;
                    });
                const contentHTML = `<html><body>${content}</body></html>`;
                await page.setContent(contentHTML);
            });
            afterEach(async () => {
                await page.close();
            });

            test('works', async () => {
                const before = await page.evaluate(isAtBottom);
                expect(before).toBe(false);

                await puppeteerUtils.infiniteScroll(page, { waitForSecs: 0 });

                const after = await page.evaluate(isAtBottom);
                expect(after).toBe(true);
            });

            test('maxScrollHeight works', async () => {
                const before = await page.evaluate(isAtBottom);
                expect(before).toBe(false);

                await puppeteerUtils.infiniteScroll(page, {
                    waitForSecs: Infinity,
                    maxScrollHeight: 1000,
                    stopScrollCallback: async () => true,
                });

                const after = await page.evaluate(isAtBottom);
                // It scrolls to the bottom in the first scroll so this is correct.
                // The test passes because the Infinite waitForSecs is broken by the height requirement.
                // If it didn't, the test would time out.
                expect(after).toBe(true);
            });

            test('stopScrollCallback works', async () => {
                const before = await page.evaluate(isAtBottom);
                expect(before).toBe(false);

                await puppeteerUtils.infiniteScroll(page, {
                    waitForSecs: Infinity,
                    stopScrollCallback: async () => true,
                });

                const after = await page.evaluate(isAtBottom);
                expect(after).toBe(true);
            });
        });

        it('saveSnapshot() works', async () => {
            const openKVSSpy = vitest.spyOn(KeyValueStore, 'open');
            const browser = await launchPuppeteer(launchContext);

            try {
                const page = await browser.newPage();
                const contentHTML =
                    '<html><head></head><body><div style="border: 1px solid black">Div number: 1</div></body></html>';
                await page.setContent(contentHTML);

                const screenshot = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 60 });

                // Test saving both image and html
                const object = { setValue: vitest.fn() };
                openKVSSpy.mockResolvedValue(object as any);
                await puppeteerUtils.saveSnapshot(page, {
                    key: 'TEST',
                    keyValueStoreName: 'TEST-STORE',
                    screenshotQuality: 60,
                });

                expect(object.setValue).toBeCalledWith('TEST.jpg', screenshot, { contentType: 'image/jpeg' });
                expect(object.setValue).toBeCalledWith('TEST.html', contentHTML, { contentType: 'text/html' });
                object.setValue.mockReset();

                // Test saving only image
                await puppeteerUtils.saveSnapshot(page, { saveHtml: false });

                // Default quality is 50
                const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 50 });
                expect(object.setValue).toBeCalledWith('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' });
            } finally {
                await browser.close();
            }
        });
    });
});