Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: templates use new dataset helpers (#2059)

Jindřich Bär9369a8580875
diff --git a/packages/templates/templates/cheerio-js/src/routes.js b/packages/templates/templates/cheerio-js/src/routes.js
index 4e491815d..a7cbbf142 100644
--- a/packages/templates/templates/cheerio-js/src/routes.js
+++ b/packages/templates/templates/cheerio-js/src/routes.js
@@ -1,4 +1,4 @@
-import { Dataset, createCheerioRouter } from 'crawlee';
+import { createCheerioRouter } from 'crawlee';
export const router = createCheerioRouter();
@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});
-router.addHandler('detail', async ({ request, $, log }) => {
+router.addHandler('detail', async ({ request, $, log, pushData }) => {
const title = $('title').text();
log.info(`${title}`, { url: request.loadedUrl });
- await Dataset.pushData({
+ await pushData({
url: request.loadedUrl,
title,
});
diff --git a/packages/templates/templates/cheerio-ts/src/routes.ts b/packages/templates/templates/cheerio-ts/src/routes.ts
index 4e491815d..a7cbbf142 100644
--- a/packages/templates/templates/cheerio-ts/src/routes.ts
+++ b/packages/templates/templates/cheerio-ts/src/routes.ts
@@ -1,4 +1,4 @@
-import { Dataset, createCheerioRouter } from 'crawlee';
+import { createCheerioRouter } from 'crawlee';
export const router = createCheerioRouter();
@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});
-router.addHandler('detail', async ({ request, $, log }) => {
+router.addHandler('detail', async ({ request, $, log, pushData }) => {
const title = $('title').text();
log.info(`${title}`, { url: request.loadedUrl });
- await Dataset.pushData({
+ await pushData({
url: request.loadedUrl,
title,
});
diff --git a/packages/templates/templates/getting-started-js/src/main.js b/packages/templates/templates/getting-started-js/src/main.js
index 256fc3a57..66844e759 100644
--- a/packages/templates/templates/getting-started-js/src/main.js
+++ b/packages/templates/templates/getting-started-js/src/main.js
@@ -1,16 +1,16 @@
// For more information, see https://crawlee.dev/
-import { PlaywrightCrawler, Dataset } from 'crawlee';
+import { PlaywrightCrawler } from 'crawlee';
// PlaywrightCrawler crawls the web using a headless
// browser controlled by the Playwright library.
const crawler = new PlaywrightCrawler({
// Use the requestHandler to process each of the crawled pages.
- async requestHandler({ request, page, enqueueLinks, log }) {
+ async requestHandler({ request, page, enqueueLinks, log, pushData }) {
const title = await page.title();
log.info(`Title of ${request.loadedUrl} is '${title}'`);
// Save results as JSON to ./storage/datasets/default
- await Dataset.pushData({ title, url: request.loadedUrl });
+ await pushData({ title, url: request.loadedUrl });
// Extract links from the current page
// and add them to the crawling queue.
diff --git a/packages/templates/templates/getting-started-ts/src/main.ts b/packages/templates/templates/getting-started-ts/src/main.ts
index 256fc3a57..66844e759 100644
--- a/packages/templates/templates/getting-started-ts/src/main.ts
+++ b/packages/templates/templates/getting-started-ts/src/main.ts
@@ -1,16 +1,16 @@
// For more information, see https://crawlee.dev/
-import { PlaywrightCrawler, Dataset } from 'crawlee';
+import { PlaywrightCrawler } from 'crawlee';
// PlaywrightCrawler crawls the web using a headless
// browser controlled by the Playwright library.
const crawler = new PlaywrightCrawler({
// Use the requestHandler to process each of the crawled pages.
- async requestHandler({ request, page, enqueueLinks, log }) {
+ async requestHandler({ request, page, enqueueLinks, log, pushData }) {
const title = await page.title();
log.info(`Title of ${request.loadedUrl} is '${title}'`);
// Save results as JSON to ./storage/datasets/default
- await Dataset.pushData({ title, url: request.loadedUrl });
+ await pushData({ title, url: request.loadedUrl });
// Extract links from the current page
// and add them to the crawling queue.
diff --git a/packages/templates/templates/playwright-js/src/routes.js b/packages/templates/templates/playwright-js/src/routes.js
index 54a62847c..e2bea3cd6 100644
--- a/packages/templates/templates/playwright-js/src/routes.js
+++ b/packages/templates/templates/playwright-js/src/routes.js
@@ -1,4 +1,4 @@
-import { Dataset, createPlaywrightRouter } from 'crawlee';
+import { createPlaywrightRouter } from 'crawlee';
export const router = createPlaywrightRouter();
@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});
-router.addHandler('detail', async ({ request, page, log }) => {
+router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });
- await Dataset.pushData({
+ await pushData({
url: request.loadedUrl,
title,
});
diff --git a/packages/templates/templates/playwright-ts/src/routes.ts b/packages/templates/templates/playwright-ts/src/routes.ts
index 54a62847c..e2bea3cd6 100644
--- a/packages/templates/templates/playwright-ts/src/routes.ts
+++ b/packages/templates/templates/playwright-ts/src/routes.ts
@@ -1,4 +1,4 @@
-import { Dataset, createPlaywrightRouter } from 'crawlee';
+import { createPlaywrightRouter } from 'crawlee';
export const router = createPlaywrightRouter();
@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});
-router.addHandler('detail', async ({ request, page, log }) => {
+router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });
- await Dataset.pushData({
+ await pushData({
url: request.loadedUrl,
title,
});
diff --git a/packages/templates/templates/puppeteer-js/src/routes.js b/packages/templates/templates/puppeteer-js/src/routes.js
index 5ddfeed13..8896a306d 100644
--- a/packages/templates/templates/puppeteer-js/src/routes.js
+++ b/packages/templates/templates/puppeteer-js/src/routes.js
@@ -1,4 +1,4 @@
-import { Dataset, createPuppeteerRouter } from 'crawlee';
+import { createPuppeteerRouter } from 'crawlee';
export const router = createPuppeteerRouter();
@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});
-router.addHandler('detail', async ({ request, page, log }) => {
+router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });
- await Dataset.pushData({
+ await pushData({
url: request.loadedUrl,
title,
});
diff --git a/packages/templates/templates/puppeteer-ts/src/routes.ts b/packages/templates/templates/puppeteer-ts/src/routes.ts
index 5ddfeed13..8896a306d 100644
--- a/packages/templates/templates/puppeteer-ts/src/routes.ts
+++ b/packages/templates/templates/puppeteer-ts/src/routes.ts
@@ -1,4 +1,4 @@
-import { Dataset, createPuppeteerRouter } from 'crawlee';
+import { createPuppeteerRouter } from 'crawlee';
export const router = createPuppeteerRouter();
@@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});
-router.addHandler('detail', async ({ request, page, log }) => {
+router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });
- await Dataset.pushData({
+ await pushData({
url: request.loadedUrl,
title,
});