Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: update apify sdk in e2e tests to use the latest dev version (#1524)

Martin Adámek4631eaccd5c8
diff --git a/package-lock.json b/package-lock.json
index b7044ca86..15dc43345 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -37,7 +37,7 @@
"@types/ws": "^7.4.7",
"@typescript-eslint/eslint-plugin": "5.36.2",
"@typescript-eslint/parser": "5.36.2",
- "apify": "3.0.2",
+ "apify": "next",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.20.0",
"commitlint": "^17.1.2",
diff --git a/package.json b/package.json
index 4b473b5db..659fc39ce 100644
--- a/package.json
+++ b/package.json
@@ -75,7 +75,7 @@
"@types/ws": "^7.4.7",
"@typescript-eslint/eslint-plugin": "5.36.2",
"@typescript-eslint/parser": "5.36.2",
- "apify": "3.0.2",
+ "apify": "next",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.20.0",
"commitlint": "^17.1.2",
diff --git a/test/e2e/automatic-persist-value/actor/package.json b/test/e2e/automatic-persist-value/actor/package.json
index 261549865..00df10db2 100644
--- a/test/e2e/automatic-persist-value/actor/package.json
+++ b/test/e2e/automatic-persist-value/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Key-Value Store - Automatic Persist Value Test",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/core": "file:./packages/core",
diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjs
index 1b101d84c..405945620 100644
--- a/test/e2e/automatic-persist-value/test.mjs
+++ b/test/e2e/automatic-persist-value/test.mjs
@@ -5,13 +5,13 @@ await initialize(testActorDirname);
const { keyValueStoreItems } = await runActor(testActorDirname);
-expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');
+await expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');
const [{ name, raw }] = keyValueStoreItems;
-expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');
+await expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');
const parsed = JSON.parse(raw.toString());
-expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');
-expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');
+await expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');
+await expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json
index 6cb03761d..0bbdf4901 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json
+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Autoscaling Pool Test - Max Tasks per Minute",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/core": "file:./packages/core",
diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.json
index 9bf8d15a4..5fd64e19e 100644
--- a/test/e2e/cheerio-default-ts/actor/package.json
+++ b/test/e2e/cheerio-default-ts/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - TypeScript",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-default/actor/package.json b/test/e2e/cheerio-default/actor/package.json
index dd796e9a2..fca0812d5 100644
--- a/test/e2e/cheerio-default/actor/package.json
+++ b/test/e2e/cheerio-default/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Default",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-enqueue-links/actor/package.json b/test/e2e/cheerio-enqueue-links/actor/package.json
index 223fda623..6db5d4ce1 100644
--- a/test/e2e/cheerio-enqueue-links/actor/package.json
+++ b/test/e2e/cheerio-enqueue-links/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Enqueue Links",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json
index d70316829..eed822f2d 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Ignore SSL Errors",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-initial-cookies/actor/package.json b/test/e2e/cheerio-initial-cookies/actor/package.json
index 2698fd088..f59673850 100644
--- a/test/e2e/cheerio-initial-cookies/actor/package.json
+++ b/test/e2e/cheerio-initial-cookies/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Initial Cookies",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-max-requests/actor/package.json b/test/e2e/cheerio-max-requests/actor/package.json
index 454a0877e..7ec97453d 100644
--- a/test/e2e/cheerio-max-requests/actor/package.json
+++ b/test/e2e/cheerio-max-requests/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Max Requests Per Crawl",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-page-info/actor/package.json b/test/e2e/cheerio-page-info/actor/package.json
index 72afad441..db75feb29 100644
--- a/test/e2e/cheerio-page-info/actor/package.json
+++ b/test/e2e/cheerio-page-info/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Page Info",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json
index e6014be1a..22055923d 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Cheerio Crawler Test - Should throw on SSL Errors",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.json
index 0070966f3..d5a4b7199 100644
--- a/test/e2e/jsdom-default-ts/actor/package.json
+++ b/test/e2e/jsdom-default-ts/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "JSDOM Crawler Test - TypeScript",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/package.json b/test/e2e/playwright-chromium-experimental-containers/actor/package.json
index 5ca93dd8c..06830471e 100644
--- a/test/e2e/playwright-chromium-experimental-containers/actor/package.json
+++ b/test/e2e/playwright-chromium-experimental-containers/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Chromium - Experimental containers",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/playwright-default/actor/package.json b/test/e2e/playwright-default/actor/package.json
index af47d9962..4757e4192 100644
--- a/test/e2e/playwright-default/actor/package.json
+++ b/test/e2e/playwright-default/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Default",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/playwright-enqueue-links/actor/package.json b/test/e2e/playwright-enqueue-links/actor/package.json
index e2912a9bf..c222f97ab 100644
--- a/test/e2e/playwright-enqueue-links/actor/package.json
+++ b/test/e2e/playwright-enqueue-links/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Enqueue Links",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/package.json b/test/e2e/playwright-firefox-experimental-containers/actor/package.json
index 87fc66205..7976966a1 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/package.json
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Firefox - Experimental containers",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.json
index 4bdf776e7..da5754a0d 100644
--- a/test/e2e/playwright-initial-cookies/actor/package.json
+++ b/test/e2e/playwright-initial-cookies/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Playwright Test - Initial Cookies",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/proxy-rotation/actor/package.json b/test/e2e/proxy-rotation/actor/package.json
index c1449b6b4..2a12ff6ff 100644
--- a/test/e2e/proxy-rotation/actor/package.json
+++ b/test/e2e/proxy-rotation/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Proxy Test - Rotation",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-default/actor/package.json b/test/e2e/puppeteer-default/actor/package.json
index f049985db..22d3ab875 100644
--- a/test/e2e/puppeteer-default/actor/package.json
+++ b/test/e2e/puppeteer-default/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Default",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-enqueue-links/actor/package.json b/test/e2e/puppeteer-enqueue-links/actor/package.json
index c5593520c..7d9a285d3 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/package.json
+++ b/test/e2e/puppeteer-enqueue-links/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Enqueue Links",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json
index 95bc57cc7..b7ec43dfb 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Ignore SSL Errors",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-initial-cookies/actor/package.json b/test/e2e/puppeteer-initial-cookies/actor/package.json
index fac1db8ca..d03543860 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/package.json
+++ b/test/e2e/puppeteer-initial-cookies/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Initial Cookies",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-page-info/actor/package.json b/test/e2e/puppeteer-page-info/actor/package.json
index 5b3f80acc..06ac60fd6 100644
--- a/test/e2e/puppeteer-page-info/actor/package.json
+++ b/test/e2e/puppeteer-page-info/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Page Info",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json
index 2814e5dcd..1e27d3e46 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Store Pagination with jQuery",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-store-pagination/actor/package.json b/test/e2e/puppeteer-store-pagination/actor/package.json
index 3389bc45d..d2f967361 100644
--- a/test/e2e/puppeteer-store-pagination/actor/package.json
+++ b/test/e2e/puppeteer-store-pagination/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Store Pagination",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json
index 7297fdd79..870445bd0 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Puppeteer Test - Should throw on SSL Errors",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser": "file:./packages/browser-crawler",
diff --git a/test/e2e/request-queue-zero-concurrency/actor/package.json b/test/e2e/request-queue-zero-concurrency/actor/package.json
index 643244c75..ccc67a08a 100644
--- a/test/e2e/request-queue-zero-concurrency/actor/package.json
+++ b/test/e2e/request-queue-zero-concurrency/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Request Queue Test - Zero Concurrency",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/request-skip-navigation/actor/package.json b/test/e2e/request-skip-navigation/actor/package.json
index 38fe57048..71a651c54 100644
--- a/test/e2e/request-skip-navigation/actor/package.json
+++ b/test/e2e/request-skip-navigation/actor/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "Request Test - skipNavigation",
"dependencies": {
- "apify": "^3.0.0-beta.75",
+ "apify": "next",
"@apify/storage-local": "^2.1.0",
"@crawlee/basic": "file:./packages/basic-crawler",
"@crawlee/browser-pool": "file:./packages/browser-pool",
diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs
index 2735754e5..96427f0d1 100644
--- a/test/e2e/tools.mjs
+++ b/test/e2e/tools.mjs
@@ -117,6 +117,8 @@ export async function runActor(dirName, memory = 4096) {
} else {
console.log(e);
}
+
+ // TODO rethrow or not?
}
}