Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(e2e): resolve flaky Camoufox tests due to GitHub rate limits (#3527)

Jindřich Bär08c98cf52de5
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index afffe6489..291583b64 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -75,3 +75,4 @@ jobs:
STORAGE_IMPLEMENTATION: ${{ matrix.storage }}
APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ GITHUB_TOKEN: ${{ github.token }}
diff --git a/package.json b/package.json
index a6d470472..328666ad4 100644
--- a/package.json
+++ b/package.json
@@ -87,7 +87,7 @@
"apify-node-curl-impersonate": "^1.0.15",
"basic-auth-parser": "^0.0.2",
"body-parser": "^2.0.0",
- "camoufox-js": "^0.9.0",
+ "camoufox-js": "^0.9.3",
"commitlint": "^20.0.0",
"cross-env": "^10.0.0",
"deep-equal": "^2.0.5",
diff --git a/packages/templates/templates/camoufox-ts/Dockerfile b/packages/templates/templates/camoufox-ts/Dockerfile
index b86983f92..3216ed4ac 100644
--- a/packages/templates/templates/camoufox-ts/Dockerfile
+++ b/packages/templates/templates/camoufox-ts/Dockerfile
@@ -1,7 +1,7 @@
# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
-FROM apify/actor-node-playwright-chrome:20-1.50.1 AS builder
+FROM apify/actor-node-playwright-camoufox:20-1.50.1 AS builder
# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
@@ -19,7 +19,7 @@ COPY --chown=myuser . ./
RUN npm run build
# Create final image
-FROM apify/actor-node-playwright-chrome:20-1.50.1
+FROM apify/actor-node-playwright-camoufox:20-1.50.1
# Copy only built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist
@@ -28,8 +28,6 @@ COPY --from=builder --chown=myuser /home/myuser/dist ./dist
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./
-# Ensure we'll install Camoufox using the npm postinstall script
-ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0
# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
diff --git a/test/e2e/camoufox-cloudflare/actor/Dockerfile b/test/e2e/camoufox-cloudflare/actor/Dockerfile
index b0215803a..b6a978cd2 100644
--- a/test/e2e/camoufox-cloudflare/actor/Dockerfile
+++ b/test/e2e/camoufox-cloudflare/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM apify/actor-node-playwright-chrome:20-1.50.1-beta AS builder
+FROM apify/actor-node-playwright-camoufox:20-1.50.1-beta AS builder
COPY /packages ./packages
COPY /package*.json ./
@@ -6,12 +6,10 @@ RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional --no-audit --ignore-scripts \
&& npm update
-FROM apify/actor-node-playwright-chrome:20-1.50.1-beta
+FROM apify/actor-node-playwright-camoufox:20-1.50.1-beta
RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
-ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0
-RUN npm run postinstall
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
COPY /.actor ./.actor
diff --git a/test/e2e/camoufox-cloudflare/actor/package.json b/test/e2e/camoufox-cloudflare/actor/package.json
index b2776bb8e..f55145923 100644
--- a/test/e2e/camoufox-cloudflare/actor/package.json
+++ b/test/e2e/camoufox-cloudflare/actor/package.json
@@ -13,7 +13,7 @@
"@crawlee/playwright": "file:./packages/playwright-crawler",
"@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils",
- "camoufox-js": "^0.2.1",
+ "camoufox-js": "^0.9.3",
"playwright": "*"
},
"overrides": {
diff --git a/yarn.lock b/yarn.lock
index b5fde88a2..1900646e5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1091,7 +1091,7 @@ __metadata:
apify-node-curl-impersonate: "npm:^1.0.15"
basic-auth-parser: "npm:^0.0.2"
body-parser: "npm:^2.0.0"
- camoufox-js: "npm:^0.9.0"
+ camoufox-js: "npm:^0.9.3"
commitlint: "npm:^20.0.0"
cross-env: "npm:^10.0.0"
deep-equal: "npm:^2.0.5"
@@ -4969,26 +4969,27 @@ __metadata:
languageName: node
linkType: hard
-"camoufox-js@npm:^0.9.0":
- version: 0.9.1
- resolution: "camoufox-js@npm:0.9.1"
+"camoufox-js@npm:^0.9.3":
+ version: 0.9.3
+ resolution: "camoufox-js@npm:0.9.3"
dependencies:
adm-zip: "npm:^0.5.16"
better-sqlite3: "npm:^12.2.0"
+ cli-progress: "npm:^3.12.0"
commander: "npm:^14.0.0"
fingerprint-generator: "npm:^2.1.66"
glob: "npm:^13.0.0"
- impit: "npm:^0.9.0"
+ impit: "npm:^0.11.0"
language-tags: "npm:^2.0.1"
maxmind: "npm:^5.0.0"
- progress: "npm:^2.0.3"
+ pretty-bytes: "npm:^7.1.0"
ua-parser-js: "npm:^2.0.2"
xml2js: "npm:^0.6.2"
peerDependencies:
playwright-core: "*"
bin:
camoufox-js: dist/__main__.js
- checksum: 10c0/1dc6dd49b5e58930c5fc9cccd9f72e4040b39d348b139aa9c0b1ff95847ba8d85aff25d56e7643d0eb3ec98a9ed66c92814ad0a81a197ed6c84223519b4332f7
+ checksum: 10c0/3ddb0176cee0e7f6b8eda61fb1290a549fb683b5e5ee7e07d3f3123751899b2a3ce821acffa3a34d5300124e1b5da02c6e3769c50a8e7e8aafe10538d3ec38c6
languageName: node
linkType: hard
@@ -5192,6 +5193,15 @@ __metadata:
languageName: node
linkType: hard
+"cli-progress@npm:^3.12.0":
+ version: 3.12.0
+ resolution: "cli-progress@npm:3.12.0"
+ dependencies:
+ string-width: "npm:^4.2.3"
+ checksum: 10c0/f464cb19ebde2f3880620a2adfaeeefaec6cb15c8e610c8a659ca1047ee90d69f3bf2fdabbb1fe33ac408678e882e3e0eecdb84ab5df0edf930b269b8a72682d
+ languageName: node
+ linkType: hard
+
"cli-spinners@npm:2.6.1":
version: 2.6.1
resolution: "cli-spinners@npm:2.6.1"
@@ -8474,6 +8484,13 @@ __metadata:
languageName: node
linkType: hard
+"impit-darwin-arm64@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-darwin-arm64@npm:0.11.0"
+ conditions: os=darwin & cpu=arm64
+ languageName: node
+ linkType: hard
+
"impit-darwin-arm64@npm:0.12.0":
version: 0.12.0
resolution: "impit-darwin-arm64@npm:0.12.0"
@@ -8481,10 +8498,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-darwin-arm64@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-darwin-arm64@npm:0.9.0"
- conditions: os=darwin & cpu=arm64
+"impit-darwin-x64@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-darwin-x64@npm:0.11.0"
+ conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
@@ -8495,10 +8512,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-darwin-x64@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-darwin-x64@npm:0.9.0"
- conditions: os=darwin & cpu=x64
+"impit-linux-arm64-gnu@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-linux-arm64-gnu@npm:0.11.0"
+ conditions: os=linux & cpu=arm64 & libc=glibc
languageName: node
linkType: hard
@@ -8509,10 +8526,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-arm64-gnu@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-linux-arm64-gnu@npm:0.9.0"
- conditions: os=linux & cpu=arm64 & libc=glibc
+"impit-linux-arm64-musl@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-linux-arm64-musl@npm:0.11.0"
+ conditions: os=linux & cpu=arm64 & libc=musl
languageName: node
linkType: hard
@@ -8523,10 +8540,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-arm64-musl@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-linux-arm64-musl@npm:0.9.0"
- conditions: os=linux & cpu=arm64 & libc=musl
+"impit-linux-x64-gnu@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-linux-x64-gnu@npm:0.11.0"
+ conditions: os=linux & cpu=x64 & libc=glibc
languageName: node
linkType: hard
@@ -8537,10 +8554,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-x64-gnu@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-linux-x64-gnu@npm:0.9.0"
- conditions: os=linux & cpu=x64 & libc=glibc
+"impit-linux-x64-musl@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-linux-x64-musl@npm:0.11.0"
+ conditions: os=linux & cpu=x64 & libc=musl
languageName: node
linkType: hard
@@ -8551,10 +8568,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-linux-x64-musl@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-linux-x64-musl@npm:0.9.0"
- conditions: os=linux & cpu=x64 & libc=musl
+"impit-win32-arm64-msvc@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-win32-arm64-msvc@npm:0.11.0"
+ conditions: os=win32 & cpu=arm64
languageName: node
linkType: hard
@@ -8565,10 +8582,10 @@ __metadata:
languageName: node
linkType: hard
-"impit-win32-arm64-msvc@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-win32-arm64-msvc@npm:0.9.0"
- conditions: os=win32 & cpu=arm64
+"impit-win32-x64-msvc@npm:0.11.0":
+ version: 0.11.0
+ resolution: "impit-win32-x64-msvc@npm:0.11.0"
+ conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
@@ -8579,25 +8596,18 @@ __metadata:
languageName: node
linkType: hard
-"impit-win32-x64-msvc@npm:0.9.0":
- version: 0.9.0
- resolution: "impit-win32-x64-msvc@npm:0.9.0"
- conditions: os=win32 & cpu=x64
- languageName: node
- linkType: hard
-
-"impit@npm:^0.12.0":
- version: 0.12.0
- resolution: "impit@npm:0.12.0"
- dependencies:
- impit-darwin-arm64: "npm:0.12.0"
- impit-darwin-x64: "npm:0.12.0"
- impit-linux-arm64-gnu: "npm:0.12.0"
- impit-linux-arm64-musl: "npm:0.12.0"
- impit-linux-x64-gnu: "npm:0.12.0"
- impit-linux-x64-musl: "npm:0.12.0"
- impit-win32-arm64-msvc: "npm:0.12.0"
- impit-win32-x64-msvc: "npm:0.12.0"
+"impit@npm:^0.11.0":
+ version: 0.11.0
+ resolution: "impit@npm:0.11.0"
+ dependencies:
+ impit-darwin-arm64: "npm:0.11.0"
+ impit-darwin-x64: "npm:0.11.0"
+ impit-linux-arm64-gnu: "npm:0.11.0"
+ impit-linux-arm64-musl: "npm:0.11.0"
+ impit-linux-x64-gnu: "npm:0.11.0"
+ impit-linux-x64-musl: "npm:0.11.0"
+ impit-win32-arm64-msvc: "npm:0.11.0"
+ impit-win32-x64-msvc: "npm:0.11.0"
dependenciesMeta:
impit-darwin-arm64:
optional: true
@@ -8615,22 +8625,22 @@ __metadata:
optional: true
impit-win32-x64-msvc:
optional: true
- checksum: 10c0/24ec37b53554aedcf1438b9eca75191043732eec66f9acf29b6098a1fc3bce2ffa8aaa808090b1588d486101043eb89f4cfe933e2095b10e79179b6a0c9ecd8b
+ checksum: 10c0/dbe0286280b63f0772e9359c39be975b8348fab7b470404afc8fff6ca9f2069f29ddd550fc7237b34dad04f0fbfa6028f87a744c040c03db06771573610cbf19
languageName: node
linkType: hard
-"impit@npm:^0.9.0":
- version: 0.9.0
- resolution: "impit@npm:0.9.0"
- dependencies:
- impit-darwin-arm64: "npm:0.9.0"
- impit-darwin-x64: "npm:0.9.0"
- impit-linux-arm64-gnu: "npm:0.9.0"
- impit-linux-arm64-musl: "npm:0.9.0"
- impit-linux-x64-gnu: "npm:0.9.0"
- impit-linux-x64-musl: "npm:0.9.0"
- impit-win32-arm64-msvc: "npm:0.9.0"
- impit-win32-x64-msvc: "npm:0.9.0"
+"impit@npm:^0.12.0":
+ version: 0.12.0
+ resolution: "impit@npm:0.12.0"
+ dependencies:
+ impit-darwin-arm64: "npm:0.12.0"
+ impit-darwin-x64: "npm:0.12.0"
+ impit-linux-arm64-gnu: "npm:0.12.0"
+ impit-linux-arm64-musl: "npm:0.12.0"
+ impit-linux-x64-gnu: "npm:0.12.0"
+ impit-linux-x64-musl: "npm:0.12.0"
+ impit-win32-arm64-msvc: "npm:0.12.0"
+ impit-win32-x64-msvc: "npm:0.12.0"
dependenciesMeta:
impit-darwin-arm64:
optional: true
@@ -8648,7 +8658,7 @@ __metadata:
optional: true
impit-win32-x64-msvc:
optional: true
- checksum: 10c0/52c777a3c2a62492690bd9cd21a2d7cd22fcece51284d26efc76727692b10636c2e3072e8cdb6c7a4de7e5edd016ad1f8a8d8ecba7afdf7f43d14d0c4d434df2
+ checksum: 10c0/24ec37b53554aedcf1438b9eca75191043732eec66f9acf29b6098a1fc3bce2ffa8aaa808090b1588d486101043eb89f4cfe933e2095b10e79179b6a0c9ecd8b
languageName: node
linkType: hard
@@ -12233,6 +12243,13 @@ __metadata:
languageName: node
linkType: hard
+"pretty-bytes@npm:^7.1.0":
+ version: 7.1.0
+ resolution: "pretty-bytes@npm:7.1.0"
+ checksum: 10c0/9458f17007bbf8b61d11ef82091bfe7f87bb2f3fd7e6aa917744eb6dc709346995f698ecbf6597ac353b0d44bb7982054f7a2325f3260c9909d09aaafbdab5ca
+ languageName: node
+ linkType: hard
+
"pretty-format@npm:30.2.0":
version: 30.2.0
resolution: "pretty-format@npm:30.2.0"