Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(e2e): resolve flaky Camoufox tests due to GitHub rate limits (#3527)
Jindřich Bär08c98cf52de5
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex afffe6489..291583b64 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -75,3 +75,4 @@ jobs:STORAGE_IMPLEMENTATION: ${{ matrix.storage }}APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ GITHUB_TOKEN: ${{ github.token }}diff --git a/package.json b/package.jsonindex a6d470472..328666ad4 100644--- a/package.json+++ b/package.json@@ -87,7 +87,7 @@"apify-node-curl-impersonate": "^1.0.15","basic-auth-parser": "^0.0.2","body-parser": "^2.0.0",- "camoufox-js": "^0.9.0",+ "camoufox-js": "^0.9.3","commitlint": "^20.0.0","cross-env": "^10.0.0","deep-equal": "^2.0.5",diff --git a/packages/templates/templates/camoufox-ts/Dockerfile b/packages/templates/templates/camoufox-ts/Dockerfileindex b86983f92..3216ed4ac 100644--- a/packages/templates/templates/camoufox-ts/Dockerfile+++ b/packages/templates/templates/camoufox-ts/Dockerfile@@ -1,7 +1,7 @@# Specify the base Docker image. You can read more about# the available images at https://crawlee.dev/docs/guides/docker-images# You can also use any other image from Docker Hub.-FROM apify/actor-node-playwright-chrome:20-1.50.1 AS builder+FROM apify/actor-node-playwright-camoufox:20-1.50.1 AS builder# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.@@ -19,7 +19,7 @@ COPY --chown=myuser . ./RUN npm run build# Create final image-FROM apify/actor-node-playwright-chrome:20-1.50.1+FROM apify/actor-node-playwright-camoufox:20-1.50.1# Copy only built JS files from builder imageCOPY --from=builder --chown=myuser /home/myuser/dist ./dist@@ -28,8 +28,6 @@ COPY --from=builder --chown=myuser /home/myuser/dist ./dist# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Ensure we'll install Camoufox using the npm postinstall script-ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingdiff --git a/test/e2e/camoufox-cloudflare/actor/Dockerfile b/test/e2e/camoufox-cloudflare/actor/Dockerfileindex b0215803a..b6a978cd2 100644--- a/test/e2e/camoufox-cloudflare/actor/Dockerfile+++ b/test/e2e/camoufox-cloudflare/actor/Dockerfile@@ -1,4 +1,4 @@-FROM apify/actor-node-playwright-chrome:20-1.50.1-beta AS builder+FROM apify/actor-node-playwright-camoufox:20-1.50.1-beta AS builderCOPY /packages ./packagesCOPY /package*.json ./@@ -6,12 +6,10 @@ RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional --no-audit --ignore-scripts \&& npm update-FROM apify/actor-node-playwright-chrome:20-1.50.1-beta+FROM apify/actor-node-playwright-camoufox:20-1.50.1-betaRUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modules-ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0-RUN npm run postinstallCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./COPY /.actor ./.actordiff --git a/test/e2e/camoufox-cloudflare/actor/package.json b/test/e2e/camoufox-cloudflare/actor/package.jsonindex b2776bb8e..f55145923 100644--- a/test/e2e/camoufox-cloudflare/actor/package.json+++ b/test/e2e/camoufox-cloudflare/actor/package.json@@ -13,7 +13,7 @@"@crawlee/playwright": "file:./packages/playwright-crawler","@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils",- "camoufox-js": "^0.2.1",+ "camoufox-js": "^0.9.3","playwright": "*"},"overrides": {diff --git a/yarn.lock b/yarn.lockindex b5fde88a2..1900646e5 100644--- a/yarn.lock+++ b/yarn.lock@@ -1091,7 +1091,7 @@ __metadata:apify-node-curl-impersonate: "npm:^1.0.15"basic-auth-parser: "npm:^0.0.2"body-parser: "npm:^2.0.0"- camoufox-js: "npm:^0.9.0"+ camoufox-js: "npm:^0.9.3"commitlint: "npm:^20.0.0"cross-env: "npm:^10.0.0"deep-equal: "npm:^2.0.5"@@ -4969,26 +4969,27 @@ __metadata:languageName: nodelinkType: hard-"camoufox-js@npm:^0.9.0":- version: 0.9.1- resolution: "camoufox-js@npm:0.9.1"+"camoufox-js@npm:^0.9.3":+ version: 0.9.3+ resolution: "camoufox-js@npm:0.9.3"dependencies:adm-zip: "npm:^0.5.16"better-sqlite3: "npm:^12.2.0"+ cli-progress: "npm:^3.12.0"commander: "npm:^14.0.0"fingerprint-generator: "npm:^2.1.66"glob: "npm:^13.0.0"- impit: "npm:^0.9.0"+ impit: "npm:^0.11.0"language-tags: "npm:^2.0.1"maxmind: "npm:^5.0.0"- progress: "npm:^2.0.3"+ pretty-bytes: "npm:^7.1.0"ua-parser-js: "npm:^2.0.2"xml2js: "npm:^0.6.2"peerDependencies:playwright-core: "*"bin:camoufox-js: dist/__main__.js- checksum: 10c0/1dc6dd49b5e58930c5fc9cccd9f72e4040b39d348b139aa9c0b1ff95847ba8d85aff25d56e7643d0eb3ec98a9ed66c92814ad0a81a197ed6c84223519b4332f7+ checksum: 10c0/3ddb0176cee0e7f6b8eda61fb1290a549fb683b5e5ee7e07d3f3123751899b2a3ce821acffa3a34d5300124e1b5da02c6e3769c50a8e7e8aafe10538d3ec38c6languageName: nodelinkType: hard@@ -5192,6 +5193,15 @@ __metadata:languageName: nodelinkType: hard+"cli-progress@npm:^3.12.0":+ version: 3.12.0+ resolution: "cli-progress@npm:3.12.0"+ dependencies:+ string-width: "npm:^4.2.3"+ checksum: 10c0/f464cb19ebde2f3880620a2adfaeeefaec6cb15c8e610c8a659ca1047ee90d69f3bf2fdabbb1fe33ac408678e882e3e0eecdb84ab5df0edf930b269b8a72682d+ languageName: node+ linkType: hard+"cli-spinners@npm:2.6.1":version: 2.6.1resolution: "cli-spinners@npm:2.6.1"@@ -8474,6 +8484,13 @@ __metadata:languageName: nodelinkType: hard+"impit-darwin-arm64@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-darwin-arm64@npm:0.11.0"+ conditions: os=darwin & cpu=arm64+ languageName: node+ linkType: hard+"impit-darwin-arm64@npm:0.12.0":version: 0.12.0resolution: "impit-darwin-arm64@npm:0.12.0"@@ -8481,10 +8498,10 @@ __metadata:languageName: nodelinkType: hard-"impit-darwin-arm64@npm:0.9.0":- version: 0.9.0- resolution: "impit-darwin-arm64@npm:0.9.0"- conditions: os=darwin & cpu=arm64+"impit-darwin-x64@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-darwin-x64@npm:0.11.0"+ conditions: os=darwin & cpu=x64languageName: nodelinkType: hard@@ -8495,10 +8512,10 @@ __metadata:languageName: nodelinkType: hard-"impit-darwin-x64@npm:0.9.0":- version: 0.9.0- resolution: "impit-darwin-x64@npm:0.9.0"- conditions: os=darwin & cpu=x64+"impit-linux-arm64-gnu@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-linux-arm64-gnu@npm:0.11.0"+ conditions: os=linux & cpu=arm64 & libc=glibclanguageName: nodelinkType: hard@@ -8509,10 +8526,10 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-arm64-gnu@npm:0.9.0":- version: 0.9.0- resolution: "impit-linux-arm64-gnu@npm:0.9.0"- conditions: os=linux & cpu=arm64 & libc=glibc+"impit-linux-arm64-musl@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-linux-arm64-musl@npm:0.11.0"+ conditions: os=linux & cpu=arm64 & libc=musllanguageName: nodelinkType: hard@@ -8523,10 +8540,10 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-arm64-musl@npm:0.9.0":- version: 0.9.0- resolution: "impit-linux-arm64-musl@npm:0.9.0"- conditions: os=linux & cpu=arm64 & libc=musl+"impit-linux-x64-gnu@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-linux-x64-gnu@npm:0.11.0"+ conditions: os=linux & cpu=x64 & libc=glibclanguageName: nodelinkType: hard@@ -8537,10 +8554,10 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-x64-gnu@npm:0.9.0":- version: 0.9.0- resolution: "impit-linux-x64-gnu@npm:0.9.0"- conditions: os=linux & cpu=x64 & libc=glibc+"impit-linux-x64-musl@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-linux-x64-musl@npm:0.11.0"+ conditions: os=linux & cpu=x64 & libc=musllanguageName: nodelinkType: hard@@ -8551,10 +8568,10 @@ __metadata:languageName: nodelinkType: hard-"impit-linux-x64-musl@npm:0.9.0":- version: 0.9.0- resolution: "impit-linux-x64-musl@npm:0.9.0"- conditions: os=linux & cpu=x64 & libc=musl+"impit-win32-arm64-msvc@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-win32-arm64-msvc@npm:0.11.0"+ conditions: os=win32 & cpu=arm64languageName: nodelinkType: hard@@ -8565,10 +8582,10 @@ __metadata:languageName: nodelinkType: hard-"impit-win32-arm64-msvc@npm:0.9.0":- version: 0.9.0- resolution: "impit-win32-arm64-msvc@npm:0.9.0"- conditions: os=win32 & cpu=arm64+"impit-win32-x64-msvc@npm:0.11.0":+ version: 0.11.0+ resolution: "impit-win32-x64-msvc@npm:0.11.0"+ conditions: os=win32 & cpu=x64languageName: nodelinkType: hard@@ -8579,25 +8596,18 @@ __metadata:languageName: nodelinkType: hard-"impit-win32-x64-msvc@npm:0.9.0":- version: 0.9.0- resolution: "impit-win32-x64-msvc@npm:0.9.0"- conditions: os=win32 & cpu=x64- languageName: node- linkType: hard--"impit@npm:^0.12.0":- version: 0.12.0- resolution: "impit@npm:0.12.0"- dependencies:- impit-darwin-arm64: "npm:0.12.0"- impit-darwin-x64: "npm:0.12.0"- impit-linux-arm64-gnu: "npm:0.12.0"- impit-linux-arm64-musl: "npm:0.12.0"- impit-linux-x64-gnu: "npm:0.12.0"- impit-linux-x64-musl: "npm:0.12.0"- impit-win32-arm64-msvc: "npm:0.12.0"- impit-win32-x64-msvc: "npm:0.12.0"+"impit@npm:^0.11.0":+ version: 0.11.0+ resolution: "impit@npm:0.11.0"+ dependencies:+ impit-darwin-arm64: "npm:0.11.0"+ impit-darwin-x64: "npm:0.11.0"+ impit-linux-arm64-gnu: "npm:0.11.0"+ impit-linux-arm64-musl: "npm:0.11.0"+ impit-linux-x64-gnu: "npm:0.11.0"+ impit-linux-x64-musl: "npm:0.11.0"+ impit-win32-arm64-msvc: "npm:0.11.0"+ impit-win32-x64-msvc: "npm:0.11.0"dependenciesMeta:impit-darwin-arm64:optional: true@@ -8615,22 +8625,22 @@ __metadata:optional: trueimpit-win32-x64-msvc:optional: true- checksum: 10c0/24ec37b53554aedcf1438b9eca75191043732eec66f9acf29b6098a1fc3bce2ffa8aaa808090b1588d486101043eb89f4cfe933e2095b10e79179b6a0c9ecd8b+ checksum: 10c0/dbe0286280b63f0772e9359c39be975b8348fab7b470404afc8fff6ca9f2069f29ddd550fc7237b34dad04f0fbfa6028f87a744c040c03db06771573610cbf19languageName: nodelinkType: hard-"impit@npm:^0.9.0":- version: 0.9.0- resolution: "impit@npm:0.9.0"- dependencies:- impit-darwin-arm64: "npm:0.9.0"- impit-darwin-x64: "npm:0.9.0"- impit-linux-arm64-gnu: "npm:0.9.0"- impit-linux-arm64-musl: "npm:0.9.0"- impit-linux-x64-gnu: "npm:0.9.0"- impit-linux-x64-musl: "npm:0.9.0"- impit-win32-arm64-msvc: "npm:0.9.0"- impit-win32-x64-msvc: "npm:0.9.0"+"impit@npm:^0.12.0":+ version: 0.12.0+ resolution: "impit@npm:0.12.0"+ dependencies:+ impit-darwin-arm64: "npm:0.12.0"+ impit-darwin-x64: "npm:0.12.0"+ impit-linux-arm64-gnu: "npm:0.12.0"+ impit-linux-arm64-musl: "npm:0.12.0"+ impit-linux-x64-gnu: "npm:0.12.0"+ impit-linux-x64-musl: "npm:0.12.0"+ impit-win32-arm64-msvc: "npm:0.12.0"+ impit-win32-x64-msvc: "npm:0.12.0"dependenciesMeta:impit-darwin-arm64:optional: true@@ -8648,7 +8658,7 @@ __metadata:optional: trueimpit-win32-x64-msvc:optional: true- checksum: 10c0/52c777a3c2a62492690bd9cd21a2d7cd22fcece51284d26efc76727692b10636c2e3072e8cdb6c7a4de7e5edd016ad1f8a8d8ecba7afdf7f43d14d0c4d434df2+ checksum: 10c0/24ec37b53554aedcf1438b9eca75191043732eec66f9acf29b6098a1fc3bce2ffa8aaa808090b1588d486101043eb89f4cfe933e2095b10e79179b6a0c9ecd8blanguageName: nodelinkType: hard@@ -12233,6 +12243,13 @@ __metadata:languageName: nodelinkType: hard+"pretty-bytes@npm:^7.1.0":+ version: 7.1.0+ resolution: "pretty-bytes@npm:7.1.0"+ checksum: 10c0/9458f17007bbf8b61d11ef82091bfe7f87bb2f3fd7e6aa917744eb6dc709346995f698ecbf6597ac353b0d44bb7982054f7a2325f3260c9909d09aaafbdab5ca+ languageName: node+ linkType: hard+"pretty-format@npm:30.2.0":version: 30.2.0resolution: "pretty-format@npm:30.2.0"