Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { BasicCrawler } from 'crawlee';
// Create a BasicCrawler - the simplest crawler that enables
// users to implement the crawling logic themselves.
const crawler = new BasicCrawler({
// This function will be called for each URL to crawl.
async requestHandler({ pushData, request, sendRequest, log }) {
const { url } = request;
log.info(`Processing ${url}...`);
// Fetch the page HTML via the crawlee sendRequest utility method
// By default, the method will use the current request that is being handled, so you don't have to
// provide it yourself. You can also provide a custom request if you want.
const { body } = await sendRequest();
// Store the HTML and URL to the default dataset.
await pushData({
url,
html: body,
});
},
});
// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
await crawler.addRequests([
'https://www.google.com',
'https://www.example.com',
'https://www.bing.com',
'https://www.wikipedia.com',
]);
// Run the crawler and wait for it to finish.
await crawler.run();
console.log('Crawler finished.');