Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { Server } from 'node:http';
import path from 'node:path';
import { MemoryStorageBackend, serviceLocator } from '@crawlee/core';
import { KeyValueStore, launchPlaywright, playwrightUtils, Request } from '@crawlee/playwright';
import { sleep } from '@crawlee/utils';
import type { Browser, Page } from 'playwright';
import { chromium } from 'playwright';
import { runExampleComServer } from '../shared/_helper.js';
import log from '@apify/log';
let serverAddress = 'http://localhost:';
let port: number;
let server: Server;
const launchContext = { launchOptions: { headless: true } };
beforeAll(async () => {
[server, port] = await runExampleComServer();
serverAddress += port;
});
afterAll(() => {
server.close();
});
describe('playwrightUtils', () => {
let ll: number;
beforeAll(async () => {
ll = log.getLevel();
log.setLevel(log.LEVELS.ERROR);
});
beforeEach(async () => {
serviceLocator.setStorageBackend(new MemoryStorageBackend());
});
afterAll(async () => {
log.setLevel(ll);
});
test('injectFile()', async () => {
const browser2 = await launchPlaywright(launchContext);
const survive = async (browser: Browser) => {
// Survive navigations
const page = await browser.newPage();
// @ts-expect-error
let result = await page.evaluate(() => window.injectedVariable === 42);
expect(result).toBe(false);
await playwrightUtils.injectFile(
page,
path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),
{
surviveNavigations: true,
},
);
// @ts-expect-error
result = await page.evaluate(() => window.injectedVariable);
expect(result).toBe(42);
await page.goto('about:chrome');
// @ts-expect-error
result = await page.evaluate(() => window.injectedVariable);
expect(result).toBe(42);
await page.goto(serverAddress);
// @ts-expect-error
result = await page.evaluate(() => window.injectedVariable);
expect(result).toBe(42);
};
const remove = async (browser: Browser) => {
// Remove with navigations
const page = await browser.newPage();
// @ts-expect-error
let result = await page.evaluate(() => window.injectedVariable === 42);
expect(result).toBe(false);
await page.goto('about:chrome');
// @ts-expect-error
result = await page.evaluate(() => window.injectedVariable === 42);
expect(result).toBe(false);
await playwrightUtils.injectFile(
page,
path.join(import.meta.dirname, '..', 'shared', 'data', 'inject_file.txt'),
);
// @ts-expect-error
result = await page.evaluate(() => window.injectedVariable);
expect(result).toBe(42);
await page.goto(serverAddress);
// @ts-expect-error
result = await page.evaluate(() => window.injectedVariable === 42);
expect(result).toBe(false);
};
try {
await Promise.all([survive(browser2), remove(browser2)]);
} finally {
await browser2.close();
}
});
test('injectJQuery()', async () => {
const browser = await launchPlaywright(launchContext);
try {
const page = await browser.newPage();
await page.goto('about:blank');
// NOTE: Chrome already defines window.$ as alias to document.querySelector(),
// (https://developers.google.com/web/tools/chrome-devtools/console/command-line-reference#queryselector)
const result1 = await page.evaluate(() => {
return {
// @ts-expect-error
isDefined: window.jQuery !== undefined,
};
});
expect(result1).toEqual({
isDefined: false,
});
await playwrightUtils.injectJQuery(page);
const result2 = await page.evaluate(() => {
/* global $ */
return {
// @ts-expect-error
isDefined: window.jQuery === window.$,
// @ts-expect-error
text: $('h1').text(),
};
});
expect(result2).toEqual({
isDefined: true,
text: '',
});
await page.reload();
const result3 = await page.evaluate(() => {
return {
// @ts-expect-error
isDefined: window.jQuery === window.$,
// @ts-expect-error
text: $('h1').text(),
};
});
expect(result3).toEqual({
isDefined: true,
text: '',
});
} finally {
await browser.close();
}
});
test('parseWithCheerio() works', async () => {
const browser = await launchPlaywright(launchContext);
try {
const page = await browser.newPage();
await page.goto(serverAddress);
const $ = await playwrightUtils.parseWithCheerio(page);
const title = $('h1').text().trim();
expect(title).toBe('Example Domain');
} finally {
await browser.close();
}
});
test('parseWithCheerio() iframe expansion works', async () => {
const browser = await launchPlaywright(launchContext);
try {
const page = await browser.newPage();
await page.goto(new URL('/special/outside-iframe', serverAddress).toString());
const $ = await playwrightUtils.parseWithCheerio(page);
const headings = $('h1')
.map((_, el) => $(el).text())
.get();
const titles = $('title')
.map((_, el) => $(el).text())
.get();
expect(titles).toEqual(['Outside iframe title']);
expect(headings).toEqual(['Outside iframe', 'In iframe']);
} finally {
await browser.close();
}
});
test('parseWithCheerio() iframe expansion works with Trusted Types CSP', async () => {
const browser = await launchPlaywright(launchContext);
try {
const page = await browser.newPage();
await page.goto(new URL('/special/outside-iframe-csp', serverAddress).toString());
const $ = await playwrightUtils.parseWithCheerio(page);
const headings = $('h1')
.map((_, el) => $(el).text())
.get();
expect(headings).toEqual(['Outside iframe', 'In iframe']);
} finally {
await browser.close();
}
});
describe('blockRequests()', () => {
let browser: Browser = null as any;
beforeAll(async () => {
browser = await launchPlaywright(launchContext);
});
afterAll(async () => {
await browser.close();
});
test('works with default values', async () => {
const loadedUrls: string[] = [];
const page = await browser.newPage();
await playwrightUtils.blockRequests(page);
page.on('response', (response) => {
if (response.url() !== `${serverAddress}/special/resources`) {
loadedUrls.push(response.url());
}
});
await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle' });
expect(loadedUrls).toEqual([`${serverAddress}/script.js`]);
});
test('works with overridden values', async () => {
const loadedUrls: string[] = [];
const page = await browser.newPage();
await playwrightUtils.blockRequests(page, {
urlPatterns: ['.css'],
});
page.on('response', (response) => loadedUrls.push(response.url()));
await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle' });
expect(loadedUrls).toEqual(
expect.arrayContaining([
`${serverAddress}/image.png`,
`${serverAddress}/script.js`,
`${serverAddress}/image.gif`,
]),
);
});
});
test('gotoExtended() works', async () => {
const browser = await chromium.launch({ headless: true });
try {
const page = await browser.newPage();
const request = new Request({
url: `${serverAddress}/special/getDebug`,
method: 'POST',
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
payload: '{ "foo": "bar" }',
});
const response = await playwrightUtils.gotoExtended(page, request);
const { method, headers, bodyLength } = JSON.parse(await response!.text());
expect(method).toBe('POST');
expect(bodyLength).toBe(16);
expect(headers['content-type']).toBe('application/json; charset=utf-8');
} finally {
await browser.close();
}
}, 60_000);
test('gotoExtended() applies headers natively for GET requests without a payload', async () => {
const browser = await chromium.launch({ headless: true });
try {
const page = await browser.newPage();
const routeSpy = vitest.spyOn(page, 'route');
const deprecatedSpy = vitest.spyOn(log, 'deprecated');
const request = new Request({
url: `${serverAddress}/special/getDebug`,
headers: {
'User-Agent': 'Demo UA',
},
});
const response = await playwrightUtils.gotoExtended(page, request);
const { method, headers } = JSON.parse(await response!.text());
expect(method).toBe('GET');
expect(headers['user-agent']).toBe('Demo UA');
expect(routeSpy).not.toHaveBeenCalled();
expect(deprecatedSpy).not.toHaveBeenCalled();
} finally {
await browser.close();
}
}, 60_000);
describe('shadow root expansion', () => {
let browser: Browser;
beforeAll(async () => {
browser = await launchPlaywright(launchContext);
});
afterAll(async () => {
await browser.close();
});
test('no expansion with ignoreShadowRoots: true', async () => {
const page = await browser.newPage();
await page.goto(`${serverAddress}/special/shadow-root`);
const result = await playwrightUtils.parseWithCheerio(page, true);
const text = result('body').text().trim();
expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(0);
expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0);
});
test('expansion works', async () => {
const page = await browser.newPage();
await page.goto(`${serverAddress}/special/shadow-root`);
const result = await playwrightUtils.parseWithCheerio(page);
const text = result('body').text().trim();
expect([...text.matchAll(/\[GOOD]/g)]).toHaveLength(2);
expect([...text.matchAll(/\[BAD]/g)]).toHaveLength(0);
});
});
describe('infiniteScroll()', () => {
function isAtBottom() {
return window.innerHeight + window.pageYOffset >= document.body.offsetHeight;
}
let browser: Browser;
beforeAll(async () => {
browser = await chromium.launch({ headless: true });
});
afterAll(async () => {
await browser.close();
});
let page: Page;
beforeEach(async () => {
page = await browser.newPage();
let count = 0;
const content = Array(1000)
.fill(null)
.map(() => {
return `<div style="border: 1px solid black">Div number: ${count++}</div>`;
});
const contentHTML = `<html><body>${content}</body></html>`;
await page.setContent(contentHTML);
});
afterEach(async () => {
await page.close();
});
test('works', async () => {
const before = await page.evaluate(isAtBottom);
expect(before).toBe(false);
await playwrightUtils.infiniteScroll(page, { waitForSecs: 0 });
const after = await page.evaluate(isAtBottom);
expect(after).toBe(true);
});
test('maxScrollHeight works', async () => {
const before = await page.evaluate(isAtBottom);
expect(before).toBe(false);
await playwrightUtils.infiniteScroll(page, {
waitForSecs: Infinity,
maxScrollHeight: 1000,
stopScrollCallback: async () => true,
});
const after = await page.evaluate(isAtBottom);
// It scrolls to the bottom in the first scroll so this is correct.
// The test passes because the Infinite waitForSecs is broken by the height requirement.
// If it didn't, the test would time out.
expect(after).toBe(true);
});
test('stopScrollCallback works', async () => {
const before = await page.evaluate(isAtBottom);
expect(before).toBe(false);
await playwrightUtils.infiniteScroll(page, {
waitForSecs: Infinity,
stopScrollCallback: async () => true,
});
const after = await page.evaluate(isAtBottom);
expect(after).toBe(true);
});
});
test('saveSnapshot() works', async () => {
const openKVSSpy = vitest.spyOn(KeyValueStore, 'open');
const browser = await chromium.launch({ headless: true });
try {
const page = await browser.newPage();
const contentHTML =
'<html><head></head><body><div style="border: 1px solid black">Div number: 1</div></body></html>';
await page.setContent(contentHTML);
const screenshot = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 60 });
// Test saving both image and html
const object = { setValue: vitest.fn() };
openKVSSpy.mockResolvedValue(object as any);
await playwrightUtils.saveSnapshot(page, {
key: 'TEST',
keyValueStoreName: 'TEST-STORE',
screenshotQuality: 60,
});
expect(object.setValue).toBeCalledWith('TEST.jpg', screenshot, { contentType: 'image/jpeg' });
expect(object.setValue).toBeCalledWith('TEST.html', contentHTML, { contentType: 'text/html' });
object.setValue.mockReset();
// Test saving only image
await playwrightUtils.saveSnapshot(page, { saveHtml: false });
// Default quality is 50
const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 50 });
expect(object.setValue).toBeCalledWith('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' });
} finally {
await browser.close();
}
});
describe('handleCloudflareChallenge() challenge detection', () => {
const { handleCloudflareChallenge } = playwrightUtils;
let browser: Browser;
let page: Page;
// no clicking or waiting, so an unsolved challenge fails fast instead of spending ~20s in the click loop
const fastOptions = () => ({
sleepSecs: 0,
preChallengeSleepSecs: 0,
clickCallback: async () => {},
});
// markup as rendered by the current (2026) Cloudflare challenge template
const currentChallengeBody = `
<div class="main-wrapper" role="main"><div class="main-content">
<h1>example.com</h1>
<h2>Performing security verification</h2>
<div style="display: grid;"><div><div><input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-2swmi_response"></div></div></div>
</div></div>
<div class="footer" role="contentinfo"><div class="footer-inner"><div class="footer-wrapper">
<div><div class="ray-id">Ray ID: <code>a29d9a9cff4fb9e4</code></div></div>
<div class="footer-link-wrapper"><span class="footer-text">Performance and Security by Cloudflare</span></div>
</div></div></div>`;
// the older template, with the `.diagnostic-wrapper` element
const oldChallengeBody = `
<div class="main-wrapper" role="main"><div class="main-content">
<h1>example.com</h1>
<div><input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-2swmi_response"></div>
</div></div>
<div class="footer" role="contentinfo"><div class="footer-inner"><div class="footer-wrapper"><div class="diagnostic-wrapper">
<div class="ray-id">Ray ID: <code>a29d9a9cff4fb9e4</code></div>
</div></div></div></div>`;
beforeAll(async () => {
browser = await chromium.launch({ headless: true });
page = await browser.newPage();
});
afterAll(async () => {
await browser.close();
});
test('detects the current challenge markup', async () => {
await page.setContent(`<html><body>${currentChallengeBody}</body></html>`);
await expect(handleCloudflareChallenge(page, 'https://example.com', fastOptions())).rejects.toThrow(
/Blocked by Cloudflare/,
);
});
test('detects the old challenge markup', async () => {
await page.setContent(`<html><body>${oldChallengeBody}</body></html>`);
await expect(handleCloudflareChallenge(page, 'https://example.com', fastOptions())).rejects.toThrow(
/Blocked by Cloudflare/,
);
});
test('detects the challenge when the challenge page navigates itself during the check', async () => {
const url = 'https://challenge.test/';
await page.route(`${url}**`, async (route) => {
if (!route.request().url().includes('__cf_chl_tk')) {
// keeps the check busy until the navigation below commits and destroys its execution context
await route.fulfill({
contentType: 'text/html',
body: `<html><body>${currentChallengeBody}<script>
window.addEventListener('load', () => setTimeout(() => { location.href = '/?__cf_chl_tk=1'; }));
const querySelector = document.querySelector.bind(document);
document.querySelector = (selector) => {
const end = Date.now() + 1000;
while (Date.now() < end) {}
return querySelector(selector);
};
</script></body></html>`,
});
return;
}
await sleep(200);
await route.fulfill({
contentType: 'text/html',
body: `<html><body>${currentChallengeBody}</body></html>`,
});
});
const selfNavigation = page.waitForRequest((request) => request.url().includes('__cf_chl_tk'));
await page.goto(url);
await selfNavigation;
await expect(handleCloudflareChallenge(page, url, fastOptions())).rejects.toThrow(/Blocked by Cloudflare/);
await page.unrouteAll();
});
test('resolves without detection on a regular page', async () => {
await page.setContent('<html><body><h1>Welcome</h1></body></html>');
await expect(
handleCloudflareChallenge(page, 'https://example.com', fastOptions()),
).resolves.toBeUndefined();
});
});
});