Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { Dataset, launchPuppeteer } from 'crawlee';
// Launch the web browser.
const browser = await launchPuppeteer();
// Create and navigate new page
console.log('Open target page');
const page = await browser.newPage();
await page.goto('https://github.com/search/advanced');
// Fill form fields and select desired search options
console.log('Fill in search form');
await page.type('#adv_code_search input.js-advanced-search-input', 'apify-js');
await page.type('#search_from', 'apify');
await page.type('#search_date', '>2015');
await page.select('select#search_language', 'JavaScript');
// Submit the form and wait for full load of next page
console.log('Submit search form');
await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle2' }),
page.click('#adv_code_search button[type="submit"]'),
]);
// Obtain and print list of search results
const results = await page.$$eval('[data-testid="results-list"] div.search-title > a', (nodes) =>
nodes.map((node) => ({
url: node.href,
name: node.innerText,
})),
);
console.log('Results:', results);
// Store data in default dataset
await Dataset.pushData(results);
// Close browser
await browser.close();