Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { CheerioCrawler, log, RequestQueue } from '@crawlee/cheerio';
import { Actor } from 'apify';
log.setLevel(log.LEVELS.DEBUG);
process.env.CRAWLEE_INTERNAL_TIMEOUT = '30000';
const mainOptions = {
exit: Actor.isAtHome(),
storage:
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
? new (await import('@apify/storage-local')).ApifyStorageLocal()
: undefined,
};
// RequestQueue auto-reset when stuck with requests in progress
await Actor.main(async () => {
const requestQueue = await RequestQueue.open();
await requestQueue.addRequest({ url: 'https://crawlee.dev/?q=1' });
await requestQueue.addRequest({ url: 'https://crawlee.dev/?q=2' });
const r3 = await requestQueue.addRequest({ url: 'https://crawlee.dev/?q=3' });
// trigger 0 concurrency by marking one of the requests as already in progress
requestQueue.inProgress.add(r3.requestId);
const crawler = new CheerioCrawler({
requestQueue,
async requestHandler({ request }) {
log.info(request.id);
},
});
await crawler.run();
}, mainOptions);