Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: update apify sdk in e2e tests to use the latest dev version (#1524)
Martin Adámek4631eaccd5c8
diff --git a/package-lock.json b/package-lock.jsonindex b7044ca86..15dc43345 100644--- a/package-lock.json+++ b/package-lock.json@@ -37,7 +37,7 @@"@types/ws": "^7.4.7","@typescript-eslint/eslint-plugin": "5.36.2","@typescript-eslint/parser": "5.36.2",- "apify": "3.0.2",+ "apify": "next","basic-auth-parser": "^0.0.2","body-parser": "^1.20.0","commitlint": "^17.1.2",diff --git a/package.json b/package.jsonindex 4b473b5db..659fc39ce 100644--- a/package.json+++ b/package.json@@ -75,7 +75,7 @@"@types/ws": "^7.4.7","@typescript-eslint/eslint-plugin": "5.36.2","@typescript-eslint/parser": "5.36.2",- "apify": "3.0.2",+ "apify": "next","basic-auth-parser": "^0.0.2","body-parser": "^1.20.0","commitlint": "^17.1.2",diff --git a/test/e2e/automatic-persist-value/actor/package.json b/test/e2e/automatic-persist-value/actor/package.jsonindex 261549865..00df10db2 100644--- a/test/e2e/automatic-persist-value/actor/package.json+++ b/test/e2e/automatic-persist-value/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Key-Value Store - Automatic Persist Value Test","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/core": "file:./packages/core",diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjsindex 1b101d84c..405945620 100644--- a/test/e2e/automatic-persist-value/test.mjs+++ b/test/e2e/automatic-persist-value/test.mjs@@ -5,13 +5,13 @@ await initialize(testActorDirname);const { keyValueStoreItems } = await runActor(testActorDirname);-expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');+await expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');const [{ name, raw }] = keyValueStoreItems;-expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');+await expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');const parsed = JSON.parse(raw.toString());-expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');-expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');+await expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');+await expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.jsonindex 6cb03761d..0bbdf4901 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Autoscaling Pool Test - Max Tasks per Minute","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/core": "file:./packages/core",diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.jsonindex 9bf8d15a4..5fd64e19e 100644--- a/test/e2e/cheerio-default-ts/actor/package.json+++ b/test/e2e/cheerio-default-ts/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - TypeScript","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-default/actor/package.json b/test/e2e/cheerio-default/actor/package.jsonindex dd796e9a2..fca0812d5 100644--- a/test/e2e/cheerio-default/actor/package.json+++ b/test/e2e/cheerio-default/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Default","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-enqueue-links/actor/package.json b/test/e2e/cheerio-enqueue-links/actor/package.jsonindex 223fda623..6db5d4ce1 100644--- a/test/e2e/cheerio-enqueue-links/actor/package.json+++ b/test/e2e/cheerio-enqueue-links/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Enqueue Links","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json b/test/e2e/cheerio-ignore-ssl-errors/actor/package.jsonindex d70316829..eed822f2d 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/package.json+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Ignore SSL Errors","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-initial-cookies/actor/package.json b/test/e2e/cheerio-initial-cookies/actor/package.jsonindex 2698fd088..f59673850 100644--- a/test/e2e/cheerio-initial-cookies/actor/package.json+++ b/test/e2e/cheerio-initial-cookies/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Initial Cookies","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-max-requests/actor/package.json b/test/e2e/cheerio-max-requests/actor/package.jsonindex 454a0877e..7ec97453d 100644--- a/test/e2e/cheerio-max-requests/actor/package.json+++ b/test/e2e/cheerio-max-requests/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Max Requests Per Crawl","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-page-info/actor/package.json b/test/e2e/cheerio-page-info/actor/package.jsonindex 72afad441..db75feb29 100644--- a/test/e2e/cheerio-page-info/actor/package.json+++ b/test/e2e/cheerio-page-info/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Page Info","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.jsonindex e6014be1a..22055923d 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Cheerio Crawler Test - Should throw on SSL Errors","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.jsonindex 0070966f3..d5a4b7199 100644--- a/test/e2e/jsdom-default-ts/actor/package.json+++ b/test/e2e/jsdom-default-ts/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "JSDOM Crawler Test - TypeScript","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/package.json b/test/e2e/playwright-chromium-experimental-containers/actor/package.jsonindex 5ca93dd8c..06830471e 100644--- a/test/e2e/playwright-chromium-experimental-containers/actor/package.json+++ b/test/e2e/playwright-chromium-experimental-containers/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Playwright Test - Chromium - Experimental containers","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/playwright-default/actor/package.json b/test/e2e/playwright-default/actor/package.jsonindex af47d9962..4757e4192 100644--- a/test/e2e/playwright-default/actor/package.json+++ b/test/e2e/playwright-default/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Playwright Test - Default","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/playwright-enqueue-links/actor/package.json b/test/e2e/playwright-enqueue-links/actor/package.jsonindex e2912a9bf..c222f97ab 100644--- a/test/e2e/playwright-enqueue-links/actor/package.json+++ b/test/e2e/playwright-enqueue-links/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Playwright Test - Enqueue Links","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/package.json b/test/e2e/playwright-firefox-experimental-containers/actor/package.jsonindex 87fc66205..7976966a1 100644--- a/test/e2e/playwright-firefox-experimental-containers/actor/package.json+++ b/test/e2e/playwright-firefox-experimental-containers/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Playwright Test - Firefox - Experimental containers","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.jsonindex 4bdf776e7..da5754a0d 100644--- a/test/e2e/playwright-initial-cookies/actor/package.json+++ b/test/e2e/playwright-initial-cookies/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Playwright Test - Initial Cookies","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/proxy-rotation/actor/package.json b/test/e2e/proxy-rotation/actor/package.jsonindex c1449b6b4..2a12ff6ff 100644--- a/test/e2e/proxy-rotation/actor/package.json+++ b/test/e2e/proxy-rotation/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Proxy Test - Rotation","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-default/actor/package.json b/test/e2e/puppeteer-default/actor/package.jsonindex f049985db..22d3ab875 100644--- a/test/e2e/puppeteer-default/actor/package.json+++ b/test/e2e/puppeteer-default/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Default","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-enqueue-links/actor/package.json b/test/e2e/puppeteer-enqueue-links/actor/package.jsonindex c5593520c..7d9a285d3 100644--- a/test/e2e/puppeteer-enqueue-links/actor/package.json+++ b/test/e2e/puppeteer-enqueue-links/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Enqueue Links","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.jsonindex 95bc57cc7..b7ec43dfb 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Ignore SSL Errors","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-initial-cookies/actor/package.json b/test/e2e/puppeteer-initial-cookies/actor/package.jsonindex fac1db8ca..d03543860 100644--- a/test/e2e/puppeteer-initial-cookies/actor/package.json+++ b/test/e2e/puppeteer-initial-cookies/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Initial Cookies","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-page-info/actor/package.json b/test/e2e/puppeteer-page-info/actor/package.jsonindex 5b3f80acc..06ac60fd6 100644--- a/test/e2e/puppeteer-page-info/actor/package.json+++ b/test/e2e/puppeteer-page-info/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Page Info","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json b/test/e2e/puppeteer-store-pagination-jquery/actor/package.jsonindex 2814e5dcd..1e27d3e46 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/package.json+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Store Pagination with jQuery","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-store-pagination/actor/package.json b/test/e2e/puppeteer-store-pagination/actor/package.jsonindex 3389bc45d..d2f967361 100644--- a/test/e2e/puppeteer-store-pagination/actor/package.json+++ b/test/e2e/puppeteer-store-pagination/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Store Pagination","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.jsonindex 7297fdd79..870445bd0 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Puppeteer Test - Should throw on SSL Errors","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser": "file:./packages/browser-crawler",diff --git a/test/e2e/request-queue-zero-concurrency/actor/package.json b/test/e2e/request-queue-zero-concurrency/actor/package.jsonindex 643244c75..ccc67a08a 100644--- a/test/e2e/request-queue-zero-concurrency/actor/package.json+++ b/test/e2e/request-queue-zero-concurrency/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Request Queue Test - Zero Concurrency","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/request-skip-navigation/actor/package.json b/test/e2e/request-skip-navigation/actor/package.jsonindex 38fe57048..71a651c54 100644--- a/test/e2e/request-skip-navigation/actor/package.json+++ b/test/e2e/request-skip-navigation/actor/package.json@@ -3,7 +3,7 @@"version": "0.0.1","description": "Request Test - skipNavigation","dependencies": {- "apify": "^3.0.0-beta.75",+ "apify": "next","@apify/storage-local": "^2.1.0","@crawlee/basic": "file:./packages/basic-crawler","@crawlee/browser-pool": "file:./packages/browser-pool",diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex 2735754e5..96427f0d1 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -117,6 +117,8 @@ export async function runActor(dirName, memory = 4096) {} else {console.log(e);}++ // TODO rethrow or not?}}