Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix cis (#2188)
Vlad Frangu725f42975f38
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.ymlindex 8650bb214..17457ee15 100644--- a/.github/workflows/docs.yml+++ b/.github/workflows/docs.yml@@ -10,59 +10,60 @@ jobs:runs-on: ubuntu-lateststeps:- - uses: actions/checkout@v4- - name: Use Node.js 18- uses: actions/setup-node@v2-beta- with:- node-version: 18+ - uses: actions/checkout@v4- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate+ - name: Use Node.js 20+ uses: actions/setup-node@v4+ with:+ node-version: 20- - name: Set git identity- run: |- git config --global user.name "Martin Adámek"- git config --global user.email "[email protected]"+ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate- - name: Build & deploy docs- run: |- # install project deps- yarn- # go to website dir- cd website- # install website deps- yarn- # build and deploy the docs- yarn deploy- env:- GIT_USER: "B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}- APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}+ - name: Set git identity+ run: |+ git config --global user.name "Martin Adámek"+ git config --global user.email "[email protected]"++ - name: Build & deploy docs+ run: |+ # install project deps+ yarn+ # go to website dir+ cd website+ # install website deps+ yarn+ # build and deploy the docs+ yarn deploy+ env:+ GIT_USER: 'B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}squash:runs-on: ubuntu-latest- needs: [ build ]+ needs: [build]steps:- - uses: actions/checkout@v4- with:- ref: 'gh-pages'- fetch-depth: 0+ - uses: actions/checkout@v4+ with:+ ref: 'gh-pages'+ fetch-depth: 0- - name: Set git identity- run: |- git config --global user.name "Martin Adámek"- git config --global user.email "[email protected]"+ - name: Set git identity+ run: |+ git config --global user.name "Martin Adámek"+ git config --global user.email "[email protected]"- - name: Squash all commits to reduce repository size- run: |- TOTAL_COMMITS="$(($(git rev-list --count HEAD)-1))"- echo "squashing $TOTAL_COMMITS commits"- git reset HEAD~$TOTAL_COMMITS- git add .- git commit --amend -m 'crawlee.dev docs'- git push --force origin gh-pages- env:- GIT_USER: "B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ - name: Squash all commits to reduce repository size+ run: |+ TOTAL_COMMITS="$(($(git rev-list --count HEAD)-1))"+ echo "squashing $TOTAL_COMMITS commits"+ git reset HEAD~$TOTAL_COMMITS+ git add .+ git commit --amend -m 'crawlee.dev docs'+ git push --force origin gh-pages+ env:+ GIT_USER: 'B4nan:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 74b672bf0..f0df325ee 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -46,13 +46,17 @@ jobs:uses: actions/setup-node@v4with:node-version: ${{ matrix.node-version }}- cache: 'yarn'- name: Enable corepackrun: |corepack enablecorepack prepare yarn@stable --activate+ - name: Activate cache for Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'+- name: Setup Playwrightuses: microsoft/playwright-github-action@v1@@ -87,17 +91,21 @@ jobs:token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}fetch-depth: 0- - name: Use Node.js 18+ - name: Use Node.js 20uses: actions/setup-node@v4with:- node-version: 18- cache: 'yarn'+ node-version: 20- name: Enable corepackrun: |corepack enablecorepack prepare yarn@stable --activate+ - name: Activate cache for Node.js 20+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'+- name: Turbo cacheid: turbo-cacheuses: actions/cache@v3diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex 126e81d21..a7a87fdc8 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -37,13 +37,17 @@ jobs:uses: actions/setup-node@v4with:node-version: ${{ matrix.node-version }}- cache: 'yarn'- name: Enable corepackrun: |corepack enablecorepack prepare yarn@stable --activate+ - name: Activate cache for Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'+- uses: microsoft/playwright-github-action@v1- name: Turbo cache@@ -72,17 +76,21 @@ jobs:- name: Checkout Source codeuses: actions/checkout@v4- - name: Use Node.js 18+ - name: Use Node.js 20uses: actions/setup-node@v4with:- node-version: 18- cache: 'yarn'+ node-version: 20- name: Enable corepackrun: |corepack enablecorepack prepare yarn@stable --activate+ - name: Activate cache for Node.js 20+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'+- name: Turbo cacheid: turbo-cacheuses: actions/cache@v3@@ -111,17 +119,21 @@ jobs:- name: Checkout repositoryuses: actions/checkout@v4- - name: Use Node.js 18+ - name: Use Node.js 20uses: actions/setup-node@v4with:- node-version: 18- cache: 'yarn'+ node-version: 20- name: Enable corepackrun: |corepack enablecorepack prepare yarn@stable --activate+ - name: Activate cache for Node.js 20+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'+- name: Turbo cacheid: turbo-cacheuses: actions/cache@v3@@ -147,17 +159,21 @@ jobs:- name: Checkout repositoryuses: actions/checkout@v4- - name: Use Node.js 18+ - name: Use Node.js 20uses: actions/setup-node@v4with:- node-version: 18- cache: 'yarn'+ node-version: 20- name: Enable corepackrun: |corepack enablecorepack prepare yarn@stable --activate+ - name: Activate cache for Node.js 20+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'+- name: Turbo cacheid: turbo-cacheuses: actions/cache@v3diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex 747821c9e..ede2693da 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -15,50 +15,53 @@ jobs:strategy:fail-fast: falsematrix:- storage: [ LOCAL, MEMORY, PLATFORM ]+ storage: [LOCAL, MEMORY, PLATFORM]steps:- - name: Cancel Workflow Action- uses: styfle/[email protected]- with:- access_token: ${{ github.token }}+ - name: Cancel Workflow Action+ uses: styfle/[email protected]+ with:+ access_token: ${{ github.token }}- - name: Checkout repository- uses: actions/checkout@v4+ - name: Checkout repository+ uses: actions/checkout@v4- - name: Use Node.js 18- uses: actions/setup-node@v4- with:- node-version: 18- cache: 'yarn'- cache-dependency-path: 'yarn.lock'+ - name: Use Node.js 20+ uses: actions/setup-node@v4+ with:+ node-version: 20- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate+ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate- - uses: microsoft/playwright-github-action@v1+ - name: Activate cache for Node.js 20+ uses: actions/setup-node@v4+ with:+ cache: 'yarn'- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}-+ - uses: microsoft/playwright-github-action@v1- - name: Login to Apify- run: npx -y apify-cli@beta login -t ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}+ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-- - name: Install Dependencies- run: yarn+ - name: Login to Apify+ run: npx -y apify-cli@beta login -t ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}- - name: Build- run: yarn ci:build+ - name: Install Dependencies+ run: yarn- - name: Test with storage ${{ matrix.storage }}- run: yarn test:e2e- env:- STORAGE_IMPLEMENTATION: ${{ matrix.storage }}+ - name: Build+ run: yarn ci:build++ - name: Test with storage ${{ matrix.storage }}+ run: yarn test:e2e+ env:+ STORAGE_IMPLEMENTATION: ${{ matrix.storage }}