Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: use `Dataset.pushData` instead of `Actor.pushData` in e2e tests
Martin Adámekdb50f36c2a22
diff --git a/test/e2e/cheerio-default/actor/main.js b/test/e2e/cheerio-default/actor/main.jsindex c36b0410f..381ba0eb9 100644--- a/test/e2e/cheerio-default/actor/main.js+++ b/test/e2e/cheerio-default/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -16,7 +16,7 @@ await Actor.main(async () => {const pageTitle = $('title').first().text();log.info(`URL: ${url} TITLE: ${pageTitle}`);- await Actor.pushData({ url, pageTitle });+ await Dataset.pushData({ url, pageTitle });},});diff --git a/test/e2e/cheerio-enqueue-links/actor/main.js b/test/e2e/cheerio-enqueue-links/actor/main.jsindex 7a542f1b2..758acb65c 100644--- a/test/e2e/cheerio-enqueue-links/actor/main.js+++ b/test/e2e/cheerio-enqueue-links/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import deepEqual from 'deep-equal';import { ApifyStorageLocal } from '@apify/storage-local';@@ -21,7 +21,7 @@ await Actor.main(async () => {if (loadedUrl.startsWith('https://drive')) {const isEqual = deepEqual(results, { processedRequests: [], unprocessedRequests: [] });- await Actor.pushData({ isEqual });+ await Dataset.pushData({ isEqual });}},});diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js b/test/e2e/cheerio-ignore-ssl-errors/actor/main.jsindex 81bcf9852..e090b72f8 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/main.js+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -22,7 +22,7 @@ await Actor.main(async () => {} else if (label === 'DETAIL') {log.info(`Scraping ${url}`);const title = $('title').text();- await Actor.pushData({ url, title });+ await Dataset.pushData({ url, title });}},});diff --git a/test/e2e/cheerio-initial-cookies/actor/main.js b/test/e2e/cheerio-initial-cookies/actor/main.jsindex 9701577ea..699b87cd3 100644--- a/test/e2e/cheerio-initial-cookies/actor/main.js+++ b/test/e2e/cheerio-initial-cookies/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';const initialCookies = [@@ -44,7 +44,7 @@ await Actor.main(async () => {}}- await Actor.pushData({ initialCookiesLength, numberOfMatchingCookies });+ await Dataset.pushData({ initialCookiesLength, numberOfMatchingCookies });},});diff --git a/test/e2e/cheerio-max-requests/actor/main.js b/test/e2e/cheerio-max-requests/actor/main.jsindex ee5e9a839..10648be9d 100644--- a/test/e2e/cheerio-max-requests/actor/main.js+++ b/test/e2e/cheerio-max-requests/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -29,7 +29,7 @@ await Actor.main(async () => {const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');- await Actor.pushData({+ await Dataset.pushData({url,uniqueIdentifier,title,diff --git a/test/e2e/cheerio-page-info/actor/main.js b/test/e2e/cheerio-page-info/actor/main.jsindex fac54855e..62f6e4c4e 100644--- a/test/e2e/cheerio-page-info/actor/main.js+++ b/test/e2e/cheerio-page-info/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler, createCheerioRouter } from '@crawlee/cheerio';+import { CheerioCrawler, createCheerioRouter, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -25,7 +25,7 @@ router.addHandler('DETAIL', async ({ request, $ }) => {const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');- await Actor.pushData({+ await Dataset.pushData({url,uniqueIdentifier,title,diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.jsindex 19b86bda5..5296d9e50 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -23,7 +23,7 @@ await Actor.main(async () => {const { url } = request;log.info(`Scraping ${url}`);const title = $('title').text();- await Actor.pushData({ url, title });+ await Dataset.pushData({ url, title });}},});diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.jsindex 5d4943dd4..5592a895b 100644--- a/test/e2e/playwright-default/actor/main.js+++ b/test/e2e/playwright-default/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PlaywrightCrawler } from '@crawlee/playwright';+import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -15,7 +15,7 @@ await Actor.main(async () => {async requestHandler({ page, enqueueLinks, request }) {const { url } = request;const pageTitle = await page.title();- await Actor.pushData({ url, pageTitle });+ await Dataset.pushData({ url, pageTitle });await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });},});diff --git a/test/e2e/proxy-rotation/actor/main.js b/test/e2e/proxy-rotation/actor/main.jsindex c83177178..ca0c82afe 100644--- a/test/e2e/proxy-rotation/actor/main.js+++ b/test/e2e/proxy-rotation/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, KeyValueStore, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -16,13 +16,13 @@ await Actor.main(async () => {const pageContent = await page.evaluate(() => document.body.children[0].innerText);const { clientIp } = JSON.parse(pageContent);- const presentAlready = await Actor.getValue(clientIp);+ const presentAlready = await KeyValueStore.getValue(clientIp);if (presentAlready) {throw new Error(`The ip address ${clientIp} was already used. Proxy rotation does not work properly.`);}- await Actor.setValue(clientIp, true);- await Actor.pushData({ clientIp });+ await KeyValueStore.setValue(clientIp, true);+ await Dataset.pushData({ clientIp });},});diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.jsindex 53283612c..291249bda 100644--- a/test/e2e/puppeteer-default/actor/main.js+++ b/test/e2e/puppeteer-default/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -15,7 +15,7 @@ await Actor.main(async () => {async requestHandler({ page, enqueueLinks, request }) {const { url } = request;const pageTitle = await page.title();- await Actor.pushData({ url, pageTitle });+ await Dataset.pushData({ url, pageTitle });await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });},});diff --git a/test/e2e/puppeteer-enqueue-links/actor/main.js b/test/e2e/puppeteer-enqueue-links/actor/main.jsindex 3b7d11dcd..a771bd632 100644--- a/test/e2e/puppeteer-enqueue-links/actor/main.js+++ b/test/e2e/puppeteer-enqueue-links/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import deepEqual from 'deep-equal';import { ApifyStorageLocal } from '@apify/storage-local';@@ -21,7 +21,7 @@ await Actor.main(async () => {if (loadedUrl.startsWith('https://drive')) {const isEqual = deepEqual(results, { processedRequests: [], unprocessedRequests: [] });- await Actor.pushData({ isEqual });+ await Dataset.pushData({ isEqual });}},});diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.jsindex 9436b2e0c..e72b15486 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -25,7 +25,7 @@ await Actor.main(async () => {} else if (label === 'DETAIL') {log.info(`Scraping ${url}`);const title = await page.title();- await Actor.pushData({ url, title });+ await Dataset.pushData({ url, title });}},});diff --git a/test/e2e/puppeteer-initial-cookies/actor/main.js b/test/e2e/puppeteer-initial-cookies/actor/main.jsindex b42d95f71..fff341937 100644--- a/test/e2e/puppeteer-initial-cookies/actor/main.js+++ b/test/e2e/puppeteer-initial-cookies/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const initialCookies = [@@ -40,7 +40,7 @@ await Actor.main(async () => {}}- await Actor.pushData({ initialCookiesLength, numberOfMatchingCookies });+ await Dataset.pushData({ initialCookiesLength, numberOfMatchingCookies });},});diff --git a/test/e2e/puppeteer-page-info/actor/main.js b/test/e2e/puppeteer-page-info/actor/main.jsindex fc2d9d714..ee853ff0e 100644--- a/test/e2e/puppeteer-page-info/actor/main.js+++ b/test/e2e/puppeteer-page-info/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -45,7 +45,7 @@ await Actor.main(async () => {const modifiedDate = new Date(Number(modifiedTimestamp));const runCount = Number(runCountText.match(/[\d,]+/)[0].replace(/,/g, ''));- await Actor.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });+ await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });}},});diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.jsindex 2caa27afb..f18f8b5dc 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -44,7 +44,7 @@ await Actor.main(async () => {runCount: Number($('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '')), // eslint-disable-line}));- await Actor.pushData({ url, uniqueIdentifier, ...results });+ await Dataset.pushData({ url, uniqueIdentifier, ...results });}},});diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.jsindex c0d991615..9357d65d7 100644--- a/test/e2e/puppeteer-store-pagination/actor/main.js+++ b/test/e2e/puppeteer-store-pagination/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';await Actor.init({ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined });@@ -53,7 +53,7 @@ crawler.router.addHandler('DETAIL', async ({ log, page, request: { url } }) => {const modifiedDate = new Date(Number(modifiedTimestamp));const runCount = Number(runCountText.match(/[\d,]+/)[0].replace(/,/g, ''));- await Actor.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });+ await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });});await crawler.run([{ url: 'https://apify.com/store?page=1', userData: { label: 'START' } }]);diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.jsindex a94c85943..86921ca39 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { PuppeteerCrawler } from '@crawlee/puppeteer';+import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';const mainOptions = {@@ -25,7 +25,7 @@ await Actor.main(async () => {} else if (label === 'DETAIL') {log.info(`Scraping ${url}`);const title = await page.title();- await Actor.pushData({ url, title });+ await Dataset.pushData({ url, title });}},});diff --git a/test/e2e/request-queue-zero-concurrency/actor/main.js b/test/e2e/request-queue-zero-concurrency/actor/main.jsindex 3d942a382..52b26b3cb 100644--- a/test/e2e/request-queue-zero-concurrency/actor/main.js+++ b/test/e2e/request-queue-zero-concurrency/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler, log } from '@crawlee/cheerio';+import { CheerioCrawler, log, RequestQueue } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';log.setLevel(log.LEVELS.DEBUG);@@ -13,7 +13,7 @@ const mainOptions = {// RequestQueue auto-reset when stuck with requests in progressawait Actor.main(async () => {- const requestQueue = await Actor.openRequestQueue();+ const requestQueue = await RequestQueue.open();await requestQueue.addRequest({ url: 'https://example.com/?q=1' });await requestQueue.addRequest({ url: 'https://example.com/?q=2' });const r3 = await requestQueue.addRequest({ url: 'https://example.com/?q=3' });diff --git a/test/e2e/request-skip-navigation/actor/main.js b/test/e2e/request-skip-navigation/actor/main.jsindex 392149acb..8c55ad6b1 100644--- a/test/e2e/request-skip-navigation/actor/main.js+++ b/test/e2e/request-skip-navigation/actor/main.js@@ -1,5 +1,5 @@import { Actor } from 'apify';-import { CheerioCrawler, log, Request } from '@crawlee/cheerio';+import { CheerioCrawler, Dataset, log, Request } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';log.setLevel(log.LEVELS.DEBUG);@@ -44,5 +44,5 @@ await Actor.main(async () => {await crawler.run([r1, r2, r3]);- await Actor.pushData({ requestCounter, navigationCounter });+ await Dataset.pushData({ requestCounter, navigationCounter });}, mainOptions);