Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: do not retry tests locally
Martin Adámek25e8bcb7f042
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex c94431163..217c83c92 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -2,6 +2,7 @@ name: Release @latestenv:YARN_IGNORE_NODE: 1+ RETRY_TESTS: 1on:workflow_dispatch:@@ -169,7 +170,7 @@ jobs:token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}repository: apify/crawleeevent-type: crawlee-docker-image-bump-+version-docs:needs: releaseruns-on: ubuntu-latest@@ -214,7 +215,7 @@ jobs:cd websiteyarn docusaurus docs:version $(jq .version ../lerna.json | sed s/\"//g | head -c 3)yarn docusaurus api:version $(jq .version ../lerna.json | sed s/\"//g | head -c 3)-+- name: Commit and push the version snapshotrun: |git config --global user.name "Apify Release Bot"diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex 524fa6530..e285f2704 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -8,6 +8,7 @@ on:env:YARN_IGNORE_NODE: 1+ RETRY_TESTS: 1jobs:# `yarn install` is done in a separate job and cached to speed up the following jobs.diff --git a/vitest.config.ts b/vitest.config.tsindex aff67df59..c10f39ce3 100644--- a/vitest.config.ts+++ b/vitest.config.ts@@ -1,8 +1,8 @@-import { cpus } from 'node:os';import { resolve } from 'node:path';import isCI from 'is-ci';import tsconfigPaths from 'vite-tsconfig-paths';+// @ts-ignoreimport { defineConfig } from 'vitest/config';let threads: { minThreads: number; maxThreads: number } | undefined;@@ -48,6 +48,6 @@ export default defineConfig({{ find: /^@crawlee\/(.*)$/, replacement: resolve(__dirname, './packages/$1/src') },{ find: /^test\/(.*)$/, replacement: resolve(__dirname, './test/$1') },],- retry: 3,+ retry: process.env.RETRY_TESTS ? 3 : 0,},});