Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { RequestQueue, CheerioCrawler } from 'crawlee';
// Open the default request queue associated with the current run
const requestQueue = await RequestQueue.open();
// Enqueue the initial requests
await requestQueue.addRequests([
{ url: 'https://example.com/1' },
{ url: 'https://example.com/2' },
{ url: 'https://example.com/3' },
// ...
]);
// The crawler will automatically process requests from the queue.
// It's used the same way for Puppeteer/Playwright crawlers
const crawler = new CheerioCrawler({
requestQueue,
async requestHandler({ enqueueLinks }) {
// Add new request to the queue
await requestQueue.addRequests([{ url: 'https://example.com/new-page' }]);
// Add links found on page to the queue
await enqueueLinks();
},
});
// Run the crawler
await crawler.run();