Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { Actor } from 'apify';
import { AdaptivePlaywrightCrawler } from '@crawlee/playwright';
import log, { LogLevel } from '@apify/log';
await Actor.init({
storage:
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
? new (await import('@apify/storage-local')).ApifyStorageLocal()
: undefined,
});
const crawler = new AdaptivePlaywrightCrawler({
// Override the comparator so that it ignores `requestHandlerMode`
resultComparator: (resultA, resultB) => {
if (resultA.datasetItems.length === 1 && resultB.datasetItems.length === 1) {
const itemA = resultA.datasetItems[0].item;
const itemB = resultB.datasetItems[0].item;
if (itemA.url === itemB.url && itemA.heading === itemB.heading) {
return 'equal';
}
}
return 'different';
},
requestHandler: async (context) => {
const { url } = context.request;
const heading = (await context.querySelectorOne('h1')).text();
const requestHandlerMode = await (async () => {
try {
await context.page.title();
return 'browser';
} catch {
return 'httpOnly';
}
})();
await context.pushData({ url, heading, requestHandlerMode });
await context.enqueueLinks({
include: ['**/next/examples/*'],
});
},
});
// CrawleeLogger has no setLevel; raise the level on the global @apify/log instead.
log.setLevel(LogLevel.DEBUG);
await crawler.run(['https://crawlee.dev/js/docs/next/examples/add-data-to-dataset']);
await Actor.exit({ exit: Actor.isAtHome() });