Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { PlaywrightCrawler, KeyValueStore } from 'crawlee';
// Create a key value store for all images we find
const imageStore = await KeyValueStore.open({ name: 'images' });
const crawler = new PlaywrightCrawler({
async requestHandler({ request, page, sendRequest }) {
// The request should have the navigation skipped
if (request.skipNavigation) {
// Request the image and get its buffer back
const imageResponse = await sendRequest();
// Saves the image in the key-value store.
//
// Note: For large-scale file downloads, consider using FileDownload crawler:
// https://crawlee.dev/js/api/http-crawler/class/FileDownload
await imageStore.setValue(`${request.userData.key}.svg`, await imageResponse.bytes(), {
contentType: 'image/svg+xml',
});
// Prevent executing the rest of the code as we do not need it
return;
}
// Get all the image sources in the current page
const images = await page.$$eval('img', (imgs) => imgs.map((img) => img.src));
// Add all the urls as requests for the crawler, giving each image a key
await crawler.addRequests(images.map((url, i) => ({ url, skipNavigation: true, userData: { key: i } })));
},
});
await crawler.addRequests(['https://crawlee.dev']);
// Run the crawler
await crawler.run();