Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { CheerioCrawler } from 'crawlee';
// Create a CheerioCrawler
const crawler = new CheerioCrawler({
// Limits the crawler to only 10 requests (do not use if you want to crawl all links)
maxRequestsPerCrawl: 10,
// Function called for each URL
async requestHandler({ request, enqueueLinks, log }) {
log.info(request.url);
// Add some links from page to the crawler's RequestQueue
await enqueueLinks({
globs: ['http?(s)://crawlee.dev/*/*'],
});
},
});
// Define the starting URL
await crawler.addRequests(['https://crawlee.dev']);
// Run the crawler
await crawler.run();