Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { CheerioCrawler } from 'crawlee';
// The crawler will automatically process requests from the queue.
// It's used the same way for Puppeteer/Playwright crawlers.
const crawler = new CheerioCrawler({
// Note that we're not specifying the requestQueue here
async requestHandler({ enqueueLinks }) {
// Add new request to the queue
await crawler.addRequests([{ url: 'https://example.com/new-page' }]);
// Add links found on page to the queue
await enqueueLinks();
},
});
// Add the initial requests.
// Note that we are not opening the request queue explicitly before
await crawler.addRequests([
{ url: 'https://example.com/1' },
{ url: 'https://example.com/2' },
{ url: 'https://example.com/3' },
// ...
]);
// Run the crawler
await crawler.run();