Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: use `Dataset.pushData` instead of `Actor.pushData` in e2e tests

Martin Adámekdb50f36c2a22
diff --git a/test/e2e/cheerio-default/actor/main.js b/test/e2e/cheerio-default/actor/main.js
index c36b0410f..381ba0eb9 100644
--- a/test/e2e/cheerio-default/actor/main.js
+++ b/test/e2e/cheerio-default/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -16,7 +16,7 @@ await Actor.main(async () => {
const pageTitle = $('title').first().text();
log.info(`URL: ${url} TITLE: ${pageTitle}`);
- await Actor.pushData({ url, pageTitle });
+ await Dataset.pushData({ url, pageTitle });
},
});
diff --git a/test/e2e/cheerio-enqueue-links/actor/main.js b/test/e2e/cheerio-enqueue-links/actor/main.js
index 7a542f1b2..758acb65c 100644
--- a/test/e2e/cheerio-enqueue-links/actor/main.js
+++ b/test/e2e/cheerio-enqueue-links/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import deepEqual from 'deep-equal';
import { ApifyStorageLocal } from '@apify/storage-local';
@@ -21,7 +21,7 @@ await Actor.main(async () => {
if (loadedUrl.startsWith('https://drive')) {
const isEqual = deepEqual(results, { processedRequests: [], unprocessedRequests: [] });
- await Actor.pushData({ isEqual });
+ await Dataset.pushData({ isEqual });
}
},
});
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js b/test/e2e/cheerio-ignore-ssl-errors/actor/main.js
index 81bcf9852..e090b72f8 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -22,7 +22,7 @@ await Actor.main(async () => {
} else if (label === 'DETAIL') {
log.info(`Scraping ${url}`);
const title = $('title').text();
- await Actor.pushData({ url, title });
+ await Dataset.pushData({ url, title });
}
},
});
diff --git a/test/e2e/cheerio-initial-cookies/actor/main.js b/test/e2e/cheerio-initial-cookies/actor/main.js
index 9701577ea..699b87cd3 100644
--- a/test/e2e/cheerio-initial-cookies/actor/main.js
+++ b/test/e2e/cheerio-initial-cookies/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
const initialCookies = [
@@ -44,7 +44,7 @@ await Actor.main(async () => {
}
}
- await Actor.pushData({ initialCookiesLength, numberOfMatchingCookies });
+ await Dataset.pushData({ initialCookiesLength, numberOfMatchingCookies });
},
});
diff --git a/test/e2e/cheerio-max-requests/actor/main.js b/test/e2e/cheerio-max-requests/actor/main.js
index ee5e9a839..10648be9d 100644
--- a/test/e2e/cheerio-max-requests/actor/main.js
+++ b/test/e2e/cheerio-max-requests/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -29,7 +29,7 @@ await Actor.main(async () => {
const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');
const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');
- await Actor.pushData({
+ await Dataset.pushData({
url,
uniqueIdentifier,
title,
diff --git a/test/e2e/cheerio-page-info/actor/main.js b/test/e2e/cheerio-page-info/actor/main.js
index fac54855e..62f6e4c4e 100644
--- a/test/e2e/cheerio-page-info/actor/main.js
+++ b/test/e2e/cheerio-page-info/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler, createCheerioRouter } from '@crawlee/cheerio';
+import { CheerioCrawler, createCheerioRouter, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -25,7 +25,7 @@ router.addHandler('DETAIL', async ({ request, $ }) => {
const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');
const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');
- await Actor.pushData({
+ await Dataset.pushData({
url,
uniqueIdentifier,
title,
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js
index 19b86bda5..5296d9e50 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -23,7 +23,7 @@ await Actor.main(async () => {
const { url } = request;
log.info(`Scraping ${url}`);
const title = $('title').text();
- await Actor.pushData({ url, title });
+ await Dataset.pushData({ url, title });
}
},
});
diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.js
index 5d4943dd4..5592a895b 100644
--- a/test/e2e/playwright-default/actor/main.js
+++ b/test/e2e/playwright-default/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PlaywrightCrawler } from '@crawlee/playwright';
+import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -15,7 +15,7 @@ await Actor.main(async () => {
async requestHandler({ page, enqueueLinks, request }) {
const { url } = request;
const pageTitle = await page.title();
- await Actor.pushData({ url, pageTitle });
+ await Dataset.pushData({ url, pageTitle });
await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });
},
});
diff --git a/test/e2e/proxy-rotation/actor/main.js b/test/e2e/proxy-rotation/actor/main.js
index c83177178..ca0c82afe 100644
--- a/test/e2e/proxy-rotation/actor/main.js
+++ b/test/e2e/proxy-rotation/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, KeyValueStore, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -16,13 +16,13 @@ await Actor.main(async () => {
const pageContent = await page.evaluate(() => document.body.children[0].innerText);
const { clientIp } = JSON.parse(pageContent);
- const presentAlready = await Actor.getValue(clientIp);
+ const presentAlready = await KeyValueStore.getValue(clientIp);
if (presentAlready) {
throw new Error(`The ip address ${clientIp} was already used. Proxy rotation does not work properly.`);
}
- await Actor.setValue(clientIp, true);
- await Actor.pushData({ clientIp });
+ await KeyValueStore.setValue(clientIp, true);
+ await Dataset.pushData({ clientIp });
},
});
diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.js
index 53283612c..291249bda 100644
--- a/test/e2e/puppeteer-default/actor/main.js
+++ b/test/e2e/puppeteer-default/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -15,7 +15,7 @@ await Actor.main(async () => {
async requestHandler({ page, enqueueLinks, request }) {
const { url } = request;
const pageTitle = await page.title();
- await Actor.pushData({ url, pageTitle });
+ await Dataset.pushData({ url, pageTitle });
await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });
},
});
diff --git a/test/e2e/puppeteer-enqueue-links/actor/main.js b/test/e2e/puppeteer-enqueue-links/actor/main.js
index 3b7d11dcd..a771bd632 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/main.js
+++ b/test/e2e/puppeteer-enqueue-links/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import deepEqual from 'deep-equal';
import { ApifyStorageLocal } from '@apify/storage-local';
@@ -21,7 +21,7 @@ await Actor.main(async () => {
if (loadedUrl.startsWith('https://drive')) {
const isEqual = deepEqual(results, { processedRequests: [], unprocessedRequests: [] });
- await Actor.pushData({ isEqual });
+ await Dataset.pushData({ isEqual });
}
},
});
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js
index 9436b2e0c..e72b15486 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -25,7 +25,7 @@ await Actor.main(async () => {
} else if (label === 'DETAIL') {
log.info(`Scraping ${url}`);
const title = await page.title();
- await Actor.pushData({ url, title });
+ await Dataset.pushData({ url, title });
}
},
});
diff --git a/test/e2e/puppeteer-initial-cookies/actor/main.js b/test/e2e/puppeteer-initial-cookies/actor/main.js
index b42d95f71..fff341937 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/main.js
+++ b/test/e2e/puppeteer-initial-cookies/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const initialCookies = [
@@ -40,7 +40,7 @@ await Actor.main(async () => {
}
}
- await Actor.pushData({ initialCookiesLength, numberOfMatchingCookies });
+ await Dataset.pushData({ initialCookiesLength, numberOfMatchingCookies });
},
});
diff --git a/test/e2e/puppeteer-page-info/actor/main.js b/test/e2e/puppeteer-page-info/actor/main.js
index fc2d9d714..ee853ff0e 100644
--- a/test/e2e/puppeteer-page-info/actor/main.js
+++ b/test/e2e/puppeteer-page-info/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -45,7 +45,7 @@ await Actor.main(async () => {
const modifiedDate = new Date(Number(modifiedTimestamp));
const runCount = Number(runCountText.match(/[\d,]+/)[0].replace(/,/g, ''));
- await Actor.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });
+ await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });
}
},
});
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
index 2caa27afb..f18f8b5dc 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -44,7 +44,7 @@ await Actor.main(async () => {
runCount: Number($('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '')), // eslint-disable-line
}));
- await Actor.pushData({ url, uniqueIdentifier, ...results });
+ await Dataset.pushData({ url, uniqueIdentifier, ...results });
}
},
});
diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.js
index c0d991615..9357d65d7 100644
--- a/test/e2e/puppeteer-store-pagination/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
await Actor.init({ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined });
@@ -53,7 +53,7 @@ crawler.router.addHandler('DETAIL', async ({ log, page, request: { url } }) => {
const modifiedDate = new Date(Number(modifiedTimestamp));
const runCount = Number(runCountText.match(/[\d,]+/)[0].replace(/,/g, ''));
- await Actor.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });
+ await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });
});
await crawler.run([{ url: 'https://apify.com/store?page=1', userData: { label: 'START' } }]);
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js
index a94c85943..86921ca39 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
const mainOptions = {
@@ -25,7 +25,7 @@ await Actor.main(async () => {
} else if (label === 'DETAIL') {
log.info(`Scraping ${url}`);
const title = await page.title();
- await Actor.pushData({ url, title });
+ await Dataset.pushData({ url, title });
}
},
});
diff --git a/test/e2e/request-queue-zero-concurrency/actor/main.js b/test/e2e/request-queue-zero-concurrency/actor/main.js
index 3d942a382..52b26b3cb 100644
--- a/test/e2e/request-queue-zero-concurrency/actor/main.js
+++ b/test/e2e/request-queue-zero-concurrency/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler, log } from '@crawlee/cheerio';
+import { CheerioCrawler, log, RequestQueue } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
log.setLevel(log.LEVELS.DEBUG);
@@ -13,7 +13,7 @@ const mainOptions = {
// RequestQueue auto-reset when stuck with requests in progress
await Actor.main(async () => {
- const requestQueue = await Actor.openRequestQueue();
+ const requestQueue = await RequestQueue.open();
await requestQueue.addRequest({ url: 'https://example.com/?q=1' });
await requestQueue.addRequest({ url: 'https://example.com/?q=2' });
const r3 = await requestQueue.addRequest({ url: 'https://example.com/?q=3' });
diff --git a/test/e2e/request-skip-navigation/actor/main.js b/test/e2e/request-skip-navigation/actor/main.js
index 392149acb..8c55ad6b1 100644
--- a/test/e2e/request-skip-navigation/actor/main.js
+++ b/test/e2e/request-skip-navigation/actor/main.js
@@ -1,5 +1,5 @@
import { Actor } from 'apify';
-import { CheerioCrawler, log, Request } from '@crawlee/cheerio';
+import { CheerioCrawler, Dataset, log, Request } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
log.setLevel(log.LEVELS.DEBUG);
@@ -44,5 +44,5 @@ await Actor.main(async () => {
await crawler.run([r1, r2, r3]);
- await Actor.pushData({ requestCounter, navigationCounter });
+ await Dataset.pushData({ requestCounter, navigationCounter });
}, mainOptions);