Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
packages:
- packages/*
- docs
- website
catalog:
playwright: 1.65.1
playwright-core: 1.65.1
zod: ^4.6.7
minimumReleaseAge: 1440
minimumReleaseAgeExclude:
- "@apify/*"
- "@crawlee/*"
- "apify-client"
- "apify"
- "crawlee"
- "got-scraping"
overrides:
"@puppeteer/browsers": "^3.0.4"
# The e2e tests run the `apify` SDK against the local 4.x workspace packages.
# The default `latest`/`next` tags are the 3.x SDK, which is not runtime
# compatible with `@crawlee/core@4` (e.g. `Configuration.INTEGER_VARS` was
# removed), so keep the SDK on the latest v4 beta and force it to use the workspace `@crawlee/*`
# packages (mirroring the per-actor `overrides.apify` used for PLATFORM tests).
apify: "^4.0.0-beta.46"
"apify>@crawlee/core": "workspace:*"
"apify>@crawlee/fs-storage": "workspace:*"
"apify>@crawlee/types": "workspace:*"
"apify>@crawlee/utils": "workspace:*"
# header-generator (via fingerprint-generator, a direct dependency of @crawlee/browser-pool)
# bundles its own [email protected], while all @crawlee/* packages depend on [email protected] directly. The two
# copies' predicate types are nominally distinct (branded symbols), so a type built with one
# doesn't satisfy a shape built with the other — this broke `@crawlee/stagehand`'s build with
# `optionsShape`/`exactShape` mismatches. header-generator's ow usage is limited to the
# long-stable basic predicate API (string/number/object.exactShape/oneOf/ofType), so it's safe
# to dedupe onto the same major version everyone else uses.
"header-generator>ow": "^3.0.0"
# A transitive playwright(-core) (stagehand) drifting from the pinned playwright breaks playwright-extra's types.
playwright: "catalog:"
playwright-core: "catalog:"
# pnpm 11 replaces `onlyBuiltDependencies` with an explicit `allowBuilds` map.
# Each entry must be true (build allowed) or false (build skipped) — pnpm 11
# refuses to install if any dep needs a build decision that isn't in the map
# (combined with `strictDepBuilds: false` below so the install can still
# proceed when new build-requesting deps appear without a manual entry).
allowBuilds:
"@apify/ui-icons": true
'@google/genai': set this to true or false
"@playwright/browser-chromium": true
"@playwright/browser-firefox": true
"@playwright/browser-webkit": true
"@swc/core": true
better-sqlite3: true
bufferutil: true
core-js: true
core-js-pure: false
esbuild: true
nx: true
protobufjs: true
puppeteer: true
unrs-resolver: true
strictDepBuilds: false
# pnpm 11 wraps every `pnpm run X` with an automatic `pnpm install` (the
# `runDepsStatusCheck` feature) so scripts always see fresh deps. The
# publish workflow runs `pnpm turbo copy --force -- --canary=major` which
# fans `pnpm run copy` out across all 23 workspace packages in parallel —
# each invocation kicks off its own `pnpm install`, they contend on the
# store and lockfile, and one of them gets killed with SIGINT, failing the
# turbo task with "command exited (1)". Disabling the auto-install keeps
# the parallel scripts deterministic; we run `pnpm install --frozen-lockfile`
# explicitly in CI before any `pnpm run` step anyway.
verifyDepsBeforeRun: false
nodeLinker: hoisted
linkWorkspacePackages: true
preferWorkspacePackages: true
publicHoistPattern:
- "*"
patchedDependencies:
'@docusaurus/[email protected]': patches/@[email protected]
'@signalwire/[email protected]': patches/@[email protected]
[email protected]: patches/[email protected]