Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: fix cis (#2188)

Vlad Frangu725f42975f38
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 8650bb214..17457ee15 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -10,59 +10,60 @@ jobs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
- - name: Use Node.js 18
- uses: actions/setup-node@v2-beta
- with:
- node-version: 18
+ - uses: actions/checkout@v4
- - name: Enable corepack
- run: |
- corepack enable
- corepack prepare yarn@stable --activate
+ - name: Use Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ node-version: 20
- - name: Set git identity
- run: |
- git config --global user.name "Martin Adámek"
- git config --global user.email "[email protected]"
+ - name: Enable corepack
+ run: |
+ corepack enable
+ corepack prepare yarn@stable --activate
- - name: Build & deploy docs
- run: |
- # install project deps
- yarn
- # go to website dir
- cd website
- # install website deps
- yarn
- # build and deploy the docs
- yarn deploy
- env:
- GIT_USER: "B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"
- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
- APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
+ - name: Set git identity
+ run: |
+ git config --global user.name "Martin Adámek"
+ git config --global user.email "[email protected]"
+
+ - name: Build & deploy docs
+ run: |
+ # install project deps
+ yarn
+ # go to website dir
+ cd website
+ # install website deps
+ yarn
+ # build and deploy the docs
+ yarn deploy
+ env:
+ GIT_USER: 'B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'
+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
squash:
runs-on: ubuntu-latest
- needs: [ build ]
+ needs: [build]
steps:
- - uses: actions/checkout@v4
- with:
- ref: 'gh-pages'
- fetch-depth: 0
+ - uses: actions/checkout@v4
+ with:
+ ref: 'gh-pages'
+ fetch-depth: 0
- - name: Set git identity
- run: |
- git config --global user.name "Martin Adámek"
- git config --global user.email "[email protected]"
+ - name: Set git identity
+ run: |
+ git config --global user.name "Martin Adámek"
+ git config --global user.email "[email protected]"
- - name: Squash all commits to reduce repository size
- run: |
- TOTAL_COMMITS="$(($(git rev-list --count HEAD)-1))"
- echo "squashing $TOTAL_COMMITS commits"
- git reset HEAD~$TOTAL_COMMITS
- git add .
- git commit --amend -m 'crawlee.dev docs'
- git push --force origin gh-pages
- env:
- GIT_USER: "B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"
- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ - name: Squash all commits to reduce repository size
+ run: |
+ TOTAL_COMMITS="$(($(git rev-list --count HEAD)-1))"
+ echo "squashing $TOTAL_COMMITS commits"
+ git reset HEAD~$TOTAL_COMMITS
+ git add .
+ git commit --amend -m 'crawlee.dev docs'
+ git push --force origin gh-pages
+ env:
+ GIT_USER: 'B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'
+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 74b672bf0..f0df325ee 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -46,13 +46,17 @@ jobs:
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- cache: 'yarn'
- name: Enable corepack
run: |
corepack enable
corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
+
- name: Setup Playwright
uses: microsoft/playwright-github-action@v1
@@ -87,17 +91,21 @@ jobs:
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
fetch-depth: 0
- - name: Use Node.js 18
+ - name: Use Node.js 20
uses: actions/setup-node@v4
with:
- node-version: 18
- cache: 'yarn'
+ node-version: 20
- name: Enable corepack
run: |
corepack enable
corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
+
- name: Turbo cache
id: turbo-cache
uses: actions/cache@v3
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index 126e81d21..a7a87fdc8 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -37,13 +37,17 @@ jobs:
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- cache: 'yarn'
- name: Enable corepack
run: |
corepack enable
corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
+
- uses: microsoft/playwright-github-action@v1
- name: Turbo cache
@@ -72,17 +76,21 @@ jobs:
- name: Checkout Source code
uses: actions/checkout@v4
- - name: Use Node.js 18
+ - name: Use Node.js 20
uses: actions/setup-node@v4
with:
- node-version: 18
- cache: 'yarn'
+ node-version: 20
- name: Enable corepack
run: |
corepack enable
corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
+
- name: Turbo cache
id: turbo-cache
uses: actions/cache@v3
@@ -111,17 +119,21 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4
- - name: Use Node.js 18
+ - name: Use Node.js 20
uses: actions/setup-node@v4
with:
- node-version: 18
- cache: 'yarn'
+ node-version: 20
- name: Enable corepack
run: |
corepack enable
corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
+
- name: Turbo cache
id: turbo-cache
uses: actions/cache@v3
@@ -147,17 +159,21 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4
- - name: Use Node.js 18
+ - name: Use Node.js 20
uses: actions/setup-node@v4
with:
- node-version: 18
- cache: 'yarn'
+ node-version: 20
- name: Enable corepack
run: |
corepack enable
corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
+
- name: Turbo cache
id: turbo-cache
uses: actions/cache@v3
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 747821c9e..ede2693da 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -15,50 +15,53 @@ jobs:
strategy:
fail-fast: false
matrix:
- storage: [ LOCAL, MEMORY, PLATFORM ]
+ storage: [LOCAL, MEMORY, PLATFORM]
steps:
- - name: Cancel Workflow Action
- uses: styfle/[email protected]
- with:
- access_token: ${{ github.token }}
+ - name: Cancel Workflow Action
+ uses: styfle/[email protected]
+ with:
+ access_token: ${{ github.token }}
- - name: Checkout repository
- uses: actions/checkout@v4
+ - name: Checkout repository
+ uses: actions/checkout@v4
- - name: Use Node.js 18
- uses: actions/setup-node@v4
- with:
- node-version: 18
- cache: 'yarn'
- cache-dependency-path: 'yarn.lock'
+ - name: Use Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ node-version: 20
- - name: Enable corepack
- run: |
- corepack enable
- corepack prepare yarn@stable --activate
+ - name: Enable corepack
+ run: |
+ corepack enable
+ corepack prepare yarn@stable --activate
- - uses: microsoft/playwright-github-action@v1
+ - name: Activate cache for Node.js 20
+ uses: actions/setup-node@v4
+ with:
+ cache: 'yarn'
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
+ - uses: microsoft/playwright-github-action@v1
- - name: Login to Apify
- run: npx -y apify-cli@beta login -t ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
- - name: Install Dependencies
- run: yarn
+ - name: Login to Apify
+ run: npx -y apify-cli@beta login -t ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}
- - name: Build
- run: yarn ci:build
+ - name: Install Dependencies
+ run: yarn
- - name: Test with storage ${{ matrix.storage }}
- run: yarn test:e2e
- env:
- STORAGE_IMPLEMENTATION: ${{ matrix.storage }}
+ - name: Build
+ run: yarn ci:build
+
+ - name: Test with storage ${{ matrix.storage }}
+ run: yarn test:e2e
+ env:
+ STORAGE_IMPLEMENTATION: ${{ matrix.storage }}