Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: small polishing for e2e scripts and tests (#1926)

Vlad Frangu4aabca4839f6
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 019c72300..9c034af74 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -11,10 +11,10 @@ jobs:
steps:
- uses: actions/checkout@v3
- - name: Use Node.js 16
+ - name: Use Node.js 18
uses: actions/setup-node@v2-beta
with:
- node-version: 16
+ node-version: 18
- name: Enable corepack
run: |
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5653d3232..7e6d4eb54 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -29,7 +29,7 @@ jobs:
matrix:
# We don't test on Windows as the tests are flaky
os: [ ubuntu-latest ]
- node-version: [ 16, 18 ]
+ node-version: [ 16, 18, 20 ]
runs-on: ${{ matrix.os }}
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index 3fe078a2c..906162629 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -19,7 +19,7 @@ jobs:
# tests on windows are extremely unstable
# os: [ ubuntu-latest, windows-2019 ]
os: [ ubuntu-latest ]
- node-version: [ 16, 18, 19 ]
+ node-version: [ 16, 18, 20 ]
steps:
- name: Cancel Workflow Action
@@ -215,7 +215,7 @@ jobs:
event-type: build-node-images
client-payload: >
{
- "release_tag": "next",
+ "release_tag": "beta",
"apify_version": "${{ steps.versions.outputs.apify }}",
"crawlee_version": "${{ steps.versions.outputs.crawlee }}",
"puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
diff --git a/packages/templates/templates/cheerio-js/Dockerfile b/packages/templates/templates/cheerio-js/Dockerfile
index 4018d6b99..2c140d5bf 100644
--- a/packages/templates/templates/cheerio-js/Dockerfile
+++ b/packages/templates/templates/cheerio-js/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node:16
+FROM apify/actor-node:18
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
diff --git a/packages/templates/templates/cheerio-ts/Dockerfile b/packages/templates/templates/cheerio-ts/Dockerfile
index 43815f97d..8d7d45836 100644
--- a/packages/templates/templates/cheerio-ts/Dockerfile
+++ b/packages/templates/templates/cheerio-ts/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node:16 AS builder
+FROM apify/actor-node:18 AS builder
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
@@ -19,7 +19,7 @@ COPY . ./
RUN npm run build
# Create final image
-FROM apify/actor-node:16
+FROM apify/actor-node:18
# Copy only built JS files from builder image
COPY --from=builder /usr/src/app/dist ./dist
diff --git a/packages/templates/templates/getting-started-js/Dockerfile b/packages/templates/templates/getting-started-js/Dockerfile
index afe996891..42c02ffb1 100644
--- a/packages/templates/templates/getting-started-js/Dockerfile
+++ b/packages/templates/templates/getting-started-js/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16
+FROM apify/actor-node-playwright-chrome:18
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
diff --git a/packages/templates/templates/getting-started-ts/Dockerfile b/packages/templates/templates/getting-started-ts/Dockerfile
index e6477ffc3..f23442a2b 100644
--- a/packages/templates/templates/getting-started-ts/Dockerfile
+++ b/packages/templates/templates/getting-started-ts/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16 AS builder
+FROM apify/actor-node-playwright-chrome:18 AS builder
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
@@ -19,7 +19,7 @@ COPY --chown=myuser . ./
RUN npm run build
# Create final image
-FROM apify/actor-node-playwright-chrome:16
+FROM apify/actor-node-playwright-chrome:18
# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist
diff --git a/packages/templates/templates/playwright-js/Dockerfile b/packages/templates/templates/playwright-js/Dockerfile
index 3ffe2058e..2e6e5027f 100644
--- a/packages/templates/templates/playwright-js/Dockerfile
+++ b/packages/templates/templates/playwright-js/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16
+FROM apify/actor-node-playwright-chrome:18
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
diff --git a/packages/templates/templates/playwright-ts/Dockerfile b/packages/templates/templates/playwright-ts/Dockerfile
index e6477ffc3..f23442a2b 100644
--- a/packages/templates/templates/playwright-ts/Dockerfile
+++ b/packages/templates/templates/playwright-ts/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:16 AS builder
+FROM apify/actor-node-playwright-chrome:18 AS builder
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
@@ -19,7 +19,7 @@ COPY --chown=myuser . ./
RUN npm run build
# Create final image
-FROM apify/actor-node-playwright-chrome:16
+FROM apify/actor-node-playwright-chrome:18
# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist
diff --git a/packages/templates/templates/puppeteer-js/Dockerfile b/packages/templates/templates/puppeteer-js/Dockerfile
index 80272b076..a6dcb4b12 100644
--- a/packages/templates/templates/puppeteer-js/Dockerfile
+++ b/packages/templates/templates/puppeteer-js/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-puppeteer-chrome:16
+FROM apify/actor-node-puppeteer-chrome:18
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
diff --git a/packages/templates/templates/puppeteer-ts/Dockerfile b/packages/templates/templates/puppeteer-ts/Dockerfile
index 19f6fa626..d0f9936ff 100644
--- a/packages/templates/templates/puppeteer-ts/Dockerfile
+++ b/packages/templates/templates/puppeteer-ts/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-puppeteer-chrome:16 AS builder
+FROM apify/actor-node-puppeteer-chrome:18 AS builder
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
@@ -19,7 +19,7 @@ COPY --chown=myuser . ./
RUN npm run build
# Create final image
-FROM apify/actor-node-puppeteer-chrome:16
+FROM apify/actor-node-puppeteer-chrome:18
# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist
diff --git a/packages/utils/src/internals/error_tracker.ts b/packages/utils/src/internals/error_tracker.ts
index ab6626721..4ccf01335 100644
--- a/packages/utils/src/internals/error_tracker.ts
+++ b/packages/utils/src/internals/error_tracker.ts
@@ -1,3 +1,5 @@
+import { inspect } from 'node:util';
+
/**
* Node.js Error interface
*/
@@ -220,7 +222,11 @@ const getErrorMessageGroup = (error: ErrnoException, storage: Record<string, unk
let { message } = error;
if (!message) {
- message = typeof error === 'string' ? error : `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
+ try {
+ message = typeof error === 'string' ? error : `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
+ } catch {
+ message = `Unknown error message. Received non-error object, and could not stringify it: ${inspect(error, { depth: 0 })}`;
+ }
}
if (!showFullMessage) {
diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
index 58d737169..a6840a48b 100644
--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-chrome:16-beta
+FROM apify/actor-node-playwright-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/playwright-chromium-experimental-containers/test.mjs b/test/e2e/playwright-chromium-experimental-containers/test.mjs
index 5af3d086e..ffd167ec7 100644
--- a/test/e2e/playwright-chromium-experimental-containers/test.mjs
+++ b/test/e2e/playwright-chromium-experimental-containers/test.mjs
@@ -1,4 +1,6 @@
-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';
+
+await skipTest('on hold');
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfile
index 58d737169..a6840a48b 100644
--- a/test/e2e/playwright-default/actor/Dockerfile
+++ b/test/e2e/playwright-default/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-chrome:16-beta
+FROM apify/actor-node-playwright-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfile
index 58d737169..a6840a48b 100644
--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile
+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-chrome:16-beta
+FROM apify/actor-node-playwright-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
index af65b94eb..670be182e 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-firefox:16-beta
+FROM apify/actor-node-playwright-firefox:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/playwright-firefox-experimental-containers/test.mjs b/test/e2e/playwright-firefox-experimental-containers/test.mjs
index 5af3d086e..ffd167ec7 100644
--- a/test/e2e/playwright-firefox-experimental-containers/test.mjs
+++ b/test/e2e/playwright-firefox-experimental-containers/test.mjs
@@ -1,4 +1,6 @@
-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';
+
+await skipTest('on hold');
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile
index 58d737169..a6840a48b 100644
--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile
+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-chrome:16-beta
+FROM apify/actor-node-playwright-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfile
index 58d737169..a6840a48b 100644
--- a/test/e2e/playwright-multi-run/actor/Dockerfile
+++ b/test/e2e/playwright-multi-run/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-chrome:16-beta
+FROM apify/actor-node-playwright-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/playwright-multi-run/test.mjs b/test/e2e/playwright-multi-run/test.mjs
index 9bd2759ec..3a8e741ab 100644
--- a/test/e2e/playwright-multi-run/test.mjs
+++ b/test/e2e/playwright-multi-run/test.mjs
@@ -1,4 +1,8 @@
-import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';
+import { initialize, getActorTestDir, runActor, expect, validateDataset, skipTest } from '../tools.mjs';
+
+if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {
+ await skipTest('not supported on platform');
+}
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/proxy-rotation/actor/Dockerfile
+++ b/test/e2e/proxy-rotation/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-default/actor/Dockerfile
+++ b/test/e2e/puppeteer-default/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-page-info/actor/Dockerfile
+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
index de6b194d0..127c35e14 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
@@ -6,7 +6,7 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-puppeteer-chrome:16-beta
+FROM apify/actor-node-puppeteer-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs
index 43ef510cf..1abca9624 100644
--- a/test/e2e/run.mjs
+++ b/test/e2e/run.mjs
@@ -93,6 +93,13 @@ async function run() {
console.log(`${colors.yellow(`[${dir.name}] `)}${match[2]}: ${c(match[1])}`);
}
});
+
+ worker.on('error', (err) => {
+ // If the worker emits any error, we want to exit with a non-zero code
+ failure = true;
+ console.log(`${colors.red('[fatal]')} test ${colors.yellow(`[${dir.name}]`)} failed with error: ${err}`);
+ });
+
worker.on('exit', async (code) => {
if (code === SKIPPED_TEST_CLOSE_CODE) {
console.log(`Test ${colors.yellow(`[${dir.name}]`)} was skipped`);
@@ -120,6 +127,7 @@ async function run() {
if (status === 'failure') failure = true;
});
+
await once(worker, 'exit');
}
}
diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfile
index de06663ad..a6840a48b 100644
--- a/test/e2e/session-rotation/actor/Dockerfile
+++ b/test/e2e/session-rotation/actor/Dockerfile
@@ -6,13 +6,13 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
&& npm update
-FROM apify/actor-node-playwright-chrome:16-beta
+FROM apify/actor-node-playwright-chrome:18-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/session-rotation/actor/package.json b/test/e2e/session-rotation/actor/package.json
index 56132f00f..f50451be0 100644
--- a/test/e2e/session-rotation/actor/package.json
+++ b/test/e2e/session-rotation/actor/package.json
@@ -10,10 +10,10 @@
"@crawlee/browser-pool": "file:./packages/browser-pool",
"@crawlee/core": "file:./packages/core",
"@crawlee/memory-storage": "file:./packages/memory-storage",
- "@crawlee/puppeteer": "file:./packages/puppeteer-crawler",
+ "@crawlee/playwright": "file:./packages/playwright-crawler",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils",
- "puppeteer": "*"
+ "playwright": "*"
},
"overrides": {
"apify": {
diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs
index 96d300465..cea58c867 100644
--- a/test/e2e/tools.mjs
+++ b/test/e2e/tools.mjs
@@ -81,7 +81,21 @@ export async function runActor(dirName, memory = 4096) {
const client = Actor.newClient();
await copyPackages(dirName);
- execSync('npx -y apify-cli@beta push', { cwd: dirName });
+ try {
+ execSync('npx -y apify-cli@beta push --no-prompt', { cwd: dirName });
+ } catch (err) {
+ console.error(colors.red(`Failed to push actor to the Apify platform. (signal ${colors.yellow(err.signal)})`));
+
+ if (err.stdout) {
+ console.log(colors.grey(` STDOUT: `), err.stdout);
+ }
+
+ if (err.stderr) {
+ console.log(colors.red(` STDERR: `), err.stderr);
+ }
+
+ throw err;
+ }
const actorName = await getActorName(dirName);
const { items: actors } = await client.actors().list();
@@ -89,7 +103,8 @@ export async function runActor(dirName, memory = 4096) {
const gotClient = got.extend({
retry: {
- limit: 0,
+ limit: 2,
+ statusCodes: [500, 502],
},
headers: {
'user-agent': 'crawlee e2e tests (got)',
@@ -101,20 +116,35 @@ export async function runActor(dirName, memory = 4096) {
// Do NOT use Apify Client yet!
// See https://github.com/apify/apify-client-js/issues/277
- const { data: { id: runId } } = await gotClient(`https://api.apify.com/v2/acts/${id}/runs`, {
- method: 'POST',
- searchParams: {
- token: client.token,
- memory,
- },
- headers: {
- 'content-type': contentType,
- },
- body: input,
- retry: {
- limit: 0,
- },
- }).json();
+ let runId;
+
+ try {
+ const { data: { id: foundRunId } } = await gotClient(`https://api.apify.com/v2/acts/${id}/runs`, {
+ method: 'POST',
+ searchParams: {
+ memory,
+ },
+ headers: {
+ 'content-type': contentType,
+ authorization: `Bearer ${client.token}`,
+ },
+ body: input,
+ retry: {
+ limit: 2,
+ statusCodes: [500, 502],
+ },
+ }).json();
+
+ runId = foundRunId;
+ } catch (err) {
+ console.error(colors.red(`Failed to start actor run on the Apify platform. (code ${colors.yellow(err.code)})`));
+
+ if (err.response) {
+ console.log(colors.grey(` RESPONSE: `), err.response.body || err.response.rawBody?.toString('utf-8'));
+ }
+
+ throw err;
+ }
const {
defaultKeyValueStoreId,