Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: templates use new dataset helpers (#2059)
Jindřich Bär9369a8580875
diff --git a/packages/templates/templates/cheerio-js/src/routes.js b/packages/templates/templates/cheerio-js/src/routes.jsindex 4e491815d..a7cbbf142 100644--- a/packages/templates/templates/cheerio-js/src/routes.js+++ b/packages/templates/templates/cheerio-js/src/routes.js@@ -1,4 +1,4 @@-import { Dataset, createCheerioRouter } from 'crawlee';+import { createCheerioRouter } from 'crawlee';export const router = createCheerioRouter();@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {});});-router.addHandler('detail', async ({ request, $, log }) => {+router.addHandler('detail', async ({ request, $, log, pushData }) => {const title = $('title').text();log.info(`${title}`, { url: request.loadedUrl });- await Dataset.pushData({+ await pushData({url: request.loadedUrl,title,});diff --git a/packages/templates/templates/cheerio-ts/src/routes.ts b/packages/templates/templates/cheerio-ts/src/routes.tsindex 4e491815d..a7cbbf142 100644--- a/packages/templates/templates/cheerio-ts/src/routes.ts+++ b/packages/templates/templates/cheerio-ts/src/routes.ts@@ -1,4 +1,4 @@-import { Dataset, createCheerioRouter } from 'crawlee';+import { createCheerioRouter } from 'crawlee';export const router = createCheerioRouter();@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {});});-router.addHandler('detail', async ({ request, $, log }) => {+router.addHandler('detail', async ({ request, $, log, pushData }) => {const title = $('title').text();log.info(`${title}`, { url: request.loadedUrl });- await Dataset.pushData({+ await pushData({url: request.loadedUrl,title,});diff --git a/packages/templates/templates/getting-started-js/src/main.js b/packages/templates/templates/getting-started-js/src/main.jsindex 256fc3a57..66844e759 100644--- a/packages/templates/templates/getting-started-js/src/main.js+++ b/packages/templates/templates/getting-started-js/src/main.js@@ -1,16 +1,16 @@// For more information, see https://crawlee.dev/-import { PlaywrightCrawler, Dataset } from 'crawlee';+import { PlaywrightCrawler } from 'crawlee';// PlaywrightCrawler crawls the web using a headless// browser controlled by the Playwright library.const crawler = new PlaywrightCrawler({// Use the requestHandler to process each of the crawled pages.- async requestHandler({ request, page, enqueueLinks, log }) {+ async requestHandler({ request, page, enqueueLinks, log, pushData }) {const title = await page.title();log.info(`Title of ${request.loadedUrl} is '${title}'`);// Save results as JSON to ./storage/datasets/default- await Dataset.pushData({ title, url: request.loadedUrl });+ await pushData({ title, url: request.loadedUrl });// Extract links from the current page// and add them to the crawling queue.diff --git a/packages/templates/templates/getting-started-ts/src/main.ts b/packages/templates/templates/getting-started-ts/src/main.tsindex 256fc3a57..66844e759 100644--- a/packages/templates/templates/getting-started-ts/src/main.ts+++ b/packages/templates/templates/getting-started-ts/src/main.ts@@ -1,16 +1,16 @@// For more information, see https://crawlee.dev/-import { PlaywrightCrawler, Dataset } from 'crawlee';+import { PlaywrightCrawler } from 'crawlee';// PlaywrightCrawler crawls the web using a headless// browser controlled by the Playwright library.const crawler = new PlaywrightCrawler({// Use the requestHandler to process each of the crawled pages.- async requestHandler({ request, page, enqueueLinks, log }) {+ async requestHandler({ request, page, enqueueLinks, log, pushData }) {const title = await page.title();log.info(`Title of ${request.loadedUrl} is '${title}'`);// Save results as JSON to ./storage/datasets/default- await Dataset.pushData({ title, url: request.loadedUrl });+ await pushData({ title, url: request.loadedUrl });// Extract links from the current page// and add them to the crawling queue.diff --git a/packages/templates/templates/playwright-js/src/routes.js b/packages/templates/templates/playwright-js/src/routes.jsindex 54a62847c..e2bea3cd6 100644--- a/packages/templates/templates/playwright-js/src/routes.js+++ b/packages/templates/templates/playwright-js/src/routes.js@@ -1,4 +1,4 @@-import { Dataset, createPlaywrightRouter } from 'crawlee';+import { createPlaywrightRouter } from 'crawlee';export const router = createPlaywrightRouter();@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {});});-router.addHandler('detail', async ({ request, page, log }) => {+router.addHandler('detail', async ({ request, page, log, pushData }) => {const title = await page.title();log.info(`${title}`, { url: request.loadedUrl });- await Dataset.pushData({+ await pushData({url: request.loadedUrl,title,});diff --git a/packages/templates/templates/playwright-ts/src/routes.ts b/packages/templates/templates/playwright-ts/src/routes.tsindex 54a62847c..e2bea3cd6 100644--- a/packages/templates/templates/playwright-ts/src/routes.ts+++ b/packages/templates/templates/playwright-ts/src/routes.ts@@ -1,4 +1,4 @@-import { Dataset, createPlaywrightRouter } from 'crawlee';+import { createPlaywrightRouter } from 'crawlee';export const router = createPlaywrightRouter();@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {});});-router.addHandler('detail', async ({ request, page, log }) => {+router.addHandler('detail', async ({ request, page, log, pushData }) => {const title = await page.title();log.info(`${title}`, { url: request.loadedUrl });- await Dataset.pushData({+ await pushData({url: request.loadedUrl,title,});diff --git a/packages/templates/templates/puppeteer-js/src/routes.js b/packages/templates/templates/puppeteer-js/src/routes.jsindex 5ddfeed13..8896a306d 100644--- a/packages/templates/templates/puppeteer-js/src/routes.js+++ b/packages/templates/templates/puppeteer-js/src/routes.js@@ -1,4 +1,4 @@-import { Dataset, createPuppeteerRouter } from 'crawlee';+import { createPuppeteerRouter } from 'crawlee';export const router = createPuppeteerRouter();@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {});});-router.addHandler('detail', async ({ request, page, log }) => {+router.addHandler('detail', async ({ request, page, log, pushData }) => {const title = await page.title();log.info(`${title}`, { url: request.loadedUrl });- await Dataset.pushData({+ await pushData({url: request.loadedUrl,title,});diff --git a/packages/templates/templates/puppeteer-ts/src/routes.ts b/packages/templates/templates/puppeteer-ts/src/routes.tsindex 5ddfeed13..8896a306d 100644--- a/packages/templates/templates/puppeteer-ts/src/routes.ts+++ b/packages/templates/templates/puppeteer-ts/src/routes.ts@@ -1,4 +1,4 @@-import { Dataset, createPuppeteerRouter } from 'crawlee';+import { createPuppeteerRouter } from 'crawlee';export const router = createPuppeteerRouter();@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {});});-router.addHandler('detail', async ({ request, page, log }) => {+router.addHandler('detail', async ({ request, page, log, pushData }) => {const title = await page.title();log.info(`${title}`, { url: request.loadedUrl });- await Dataset.pushData({+ await pushData({url: request.loadedUrl,title,});