Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { Server } from 'node:http';
import { sleep } from '@crawlee/utils';
import { launchPuppeteer, puppeteerUtils } from 'crawlee';
import type { HTTPRequest } from 'puppeteer';
import { runExampleComServer } from '../shared/_helper.js';
const { addInterceptRequestHandler, removeInterceptRequestHandler } = puppeteerUtils;
let serverAddress = 'http://localhost:';
let port: number;
let server: Server;
beforeAll(async () => {
[server, port] = await runExampleComServer();
serverAddress += port;
});
afterAll(() => {
server.close();
});
describe('puppeteerUtils.addInterceptRequestHandler|removeInterceptRequestHandler()', () => {
test('should allow multiple handlers', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
const allUrls: string[] = [];
const loadedUrls: string[] = [];
try {
const page = await browser.newPage();
// Just collect all URLs.
await addInterceptRequestHandler(page, async (request) => {
allUrls.push(request.url());
return request.continue();
});
// Abort images.
await addInterceptRequestHandler(page, async (request) => {
if (request.resourceType() === 'image') return request.abort();
return request.continue();
});
// Abort scripts.
await addInterceptRequestHandler(page, async (request) => {
if (request.resourceType() === 'script') return request.abort();
return request.continue();
});
// Save all loaded URLs.
page.on('response', (response) => loadedUrls.push(response.url()));
await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });
} finally {
await browser.close();
}
expect(allUrls).toEqual(
expect.arrayContaining([
`${serverAddress}/script.js`,
`${serverAddress}/style.css`,
`${serverAddress}/image.png`,
]),
);
expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`]));
});
test('should not propagate aborted/responded requests to following handlers', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
const propagatedUrls: string[] = [];
try {
const page = await browser.newPage();
// Abort images.
await addInterceptRequestHandler(page, async (request) => {
if (request.resourceType() === 'image') return request.abort();
return request.continue();
});
// Respond scripts.
await addInterceptRequestHandler(page, async (request) => {
if (request.resourceType() === 'script') {
return request.respond({
status: 404,
});
}
return request.continue();
});
// Just collect all URLs propagated to the last handler.
await addInterceptRequestHandler(page, async (request) => {
propagatedUrls.push(request.url());
return request.continue();
});
await page.goto(`${serverAddress}/special/resources`);
} finally {
await browser.close();
}
expect(propagatedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`]));
});
test('should allow to modify request', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
try {
const page = await browser.newPage();
// Change all requests to DELETE.
await addInterceptRequestHandler(page, async (request) => {
return request.continue({ method: 'DELETE' });
});
// Add some query parameters to request URLs.
await addInterceptRequestHandler(page, async (request) => {
return request.continue({
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
postData: '{ "foo": "bar" }',
});
});
// Change all requests to POST and add payload.
await addInterceptRequestHandler(page, async (request) => {
return request.continue({
method: 'POST',
});
});
// Check response that it's correct.
const response = await page.goto(`${serverAddress}/special/getDebug`, { waitUntil: 'networkidle0' });
const { method, headers, bodyLength } = JSON.parse(await response!.text());
expect(method).toBe('POST');
expect(bodyLength).toBe(16);
expect(headers['content-type']).toBe('application/json; charset=utf-8');
} finally {
await browser.close();
}
});
test('should merge headers passed as a Headers instance', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
try {
const page = await browser.newPage();
await addInterceptRequestHandler(page, async (request) => {
return request.continue({ headers: { 'X-First': 'one' } });
});
await addInterceptRequestHandler(page, async (request) => {
return request.continue({ headers: new Headers({ 'X-Second': 'two' }) });
});
const response = await page.goto(`${serverAddress}/special/getDebug`, { waitUntil: 'networkidle0' });
const { headers } = JSON.parse(await response!.text());
expect(headers['x-first']).toBe('one');
expect(headers['x-second']).toBe('two');
} finally {
await browser.close();
}
});
test('should allow async handler', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
try {
const page = await browser.newPage();
await addInterceptRequestHandler(page, async (request) => {
await sleep(100);
return request.continue({
method: 'POST',
});
});
// Check response that it's correct.
const response = await page.goto(`${serverAddress}/special/getDebug`, { waitUntil: 'networkidle0' });
const { method } = JSON.parse(await response!.text());
expect(method).toBe('POST');
} finally {
await browser.close();
}
});
describe('internal handleRequest function should return correctly formatted headers', () => {
test('should correctly capitalize headers', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
try {
const page = await browser.newPage();
await addInterceptRequestHandler(page, async (request) => {
// Override headers
const headers = {
...request.headers(),
'accept': 'text/html',
'accept-language': 'en-GB',
'upgrade-insecure-requests': '2',
};
return request.continue({ headers });
});
const response = await page.goto(`${serverAddress}/special/getRawHeaders`);
const rawHeadersArr = JSON.parse(await response!.text()) as string[];
const acceptIndex = rawHeadersArr.findIndex((headerItem) => headerItem === 'Accept');
expect(typeof acceptIndex).toBe('number');
expect(rawHeadersArr[acceptIndex + 1]).toEqual('text/html');
const acceptLanguageIndex = rawHeadersArr.findIndex((headerItem) => headerItem === 'Accept-Language');
expect(typeof acceptLanguageIndex).toBe('number');
expect(rawHeadersArr[acceptLanguageIndex + 1]).toEqual('en-GB');
const upgradeInsReqIndex = rawHeadersArr.findIndex(
(headerItem) => headerItem === 'Upgrade-Insecure-Requests',
);
expect(typeof upgradeInsReqIndex).toBe('number');
expect(rawHeadersArr[upgradeInsReqIndex + 1]).toEqual('2');
// defaults should be capitalized too
const connectionIndex = rawHeadersArr.findIndex((headerItem) => headerItem === 'Connection');
expect(typeof connectionIndex).toBe('number');
expect(rawHeadersArr[connectionIndex + 1]).toEqual('keep-alive');
} finally {
await browser.close();
}
});
});
});
describe('puppeteerUtils.removeInterceptRequestHandler()', () => {
test('works', async () => {
const browser = await launchPuppeteer({ launchOptions: { headless: true } });
const loadedUrls: string[] = [];
try {
const page = await browser.newPage();
page.on('response', (response) => loadedUrls.push(response.url()));
// Abort images.
const abortImagesHandler = async (request: HTTPRequest) => {
if (request.resourceType() === 'image') return request.abort();
return request.continue();
};
await addInterceptRequestHandler(page, abortImagesHandler);
// Abort scripts.
await addInterceptRequestHandler(page, async (request) => {
if (request.resourceType() === 'script') return request.abort();
return request.continue();
});
// Load with scripts and images disabled.
await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });
expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`]));
// Try it once again.
await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });
expect(loadedUrls).toEqual(
expect.arrayContaining([`${serverAddress}/style.css`, `${serverAddress}/style.css`]),
);
// Enable images.
await removeInterceptRequestHandler(page, abortImagesHandler);
// Try to load once again if images appear there.
await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });
expect(loadedUrls).toEqual(
expect.arrayContaining([
`${serverAddress}/style.css`,
`${serverAddress}/style.css`,
`${serverAddress}/style.css`,
`${serverAddress}/image.png`,
]),
);
} finally {
await browser.close();
}
});
});