Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

6.4 KB
Raw Blame
{
    "name": "@crawlee/root",
    "private": true,
    "type": "module",
    "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
    "keywords": [
        "apify",
        "headless",
        "chrome",
        "puppeteer",
        "crawler",
        "scraper"
    ],
    "author": {
        "name": "Apify",
        "email": "[email protected]",
        "url": "https://apify.com"
    },
    "contributors": [
        "Jan Curn <[email protected]>",
        "Marek Trunkat <[email protected]>",
        "Ondra Urban <[email protected]>"
    ],
    "license": "Apache-2.0",
    "repository": {
        "type": "git",
        "url": "git+https://github.com/apify/crawlee"
    },
    "bugs": {
        "url": "https://github.com/apify/crawlee/issues"
    },
    "homepage": "https://crawlee.dev",
    "scripts": {
        "prepublishOnly": "turbo run copy --filter=./packages/*",
        "clean": "turbo run clean --filter=./packages/* && rimraf .turbo packages/*/.turbo packages/*/*.tsbuildinfo",
        "build": "turbo run build --filter=./packages/* && node ./scripts/typescript-fixes.mjs",
        "ci:build": "turbo run build --filter=./packages/* --cache-dir=\".turbo\" && node ./scripts/typescript-fixes.mjs",
        "test": "vitest run --silent",
        "test:e2e": "node test/e2e/run.mjs",
        "test:integration": "cross-env CRAWLEE_DIFFICULT_TESTS=1 vitest run --silent=true test/integration",
        "test:integration:services:up": "docker network create crawlee-it 2>/dev/null; docker run -d --rm --name crawlee-it-browserless --network crawlee-it -p 3000:3000 -e CONCURRENT=4 ghcr.io/browserless/chromium && docker run -d --rm --name crawlee-it-httpbin --network crawlee-it --network-alias httpbin -p 8080:80 kennethreitz/httpbin",
        "test:integration:services:down": "docker stop crawlee-it-browserless crawlee-it-httpbin; docker network rm crawlee-it 2>/dev/null; true",
        "test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 CRAWLEE_TEST_BROWSERS=chromium,firefox,webkit vitest run --silent",
        "tsc-check-tests": "tsc --noEmit --project test/tsconfig.json",
        "coverage": "vitest --coverage",
        "publish:next": "lerna publish from-package --contents dist --dist-tag v4 --force-publish --yes",
        "release:next": "pnpm build && pnpm publish:next",
        "publish:rc": "lerna publish from-package --contents dist --dist-tag rc --force-publish --yes",
        "publish:prod": "lerna publish from-package --contents dist --force-publish --yes",
        "release:prod": "pnpm build && pnpm publish:prod",
        "release:pin-versions": "turbo run copy -- -- --pin-versions",
        "api:extract": "pnpm build && pnpm dlx github:apify/api-extractor-report --exclude=@crawlee/cli,@crawlee/templates --extract-command='pnpm api:extract'",
        "api:check": "pnpm dlx github:apify/api-extractor-report --exclude=@crawlee/cli,@crawlee/templates --extract-command='pnpm api:extract' --verify",
        "lint": "oxlint packages test docs --tsconfig=tsconfig.json --type-aware",
        "lint:fix": "oxlint packages test docs --tsconfig=tsconfig.json --type-aware --fix",
        "format": "oxfmt packages test docs --write",
        "format:check": "oxfmt packages test docs --check",
        "prepare": "husky"
    },
    "devDependencies": {
        "@apify/log": "^3.0.2",
        "@apify/oxlint-config": "^0.3.0",
        "@apify/tsconfig": "^0.2.0",
        "@crawlee/basic": "workspace:*",
        "@crawlee/cheerio": "workspace:*",
        "@crawlee/core": "workspace:*",
        "@crawlee/fs-storage": "workspace:*",
        "@crawlee/http-client": "workspace:*",
        "@crawlee/impit-client": "workspace:*",
        "@crawlee/otel": "workspace:*",
        "@crawlee/playwright": "workspace:*",
        "@crawlee/puppeteer": "workspace:*",
        "@crawlee/stagehand": "workspace:*",
        "@crawlee/types": "workspace:*",
        "@crawlee/utils": "workspace:*",
        "@microsoft/api-extractor": "^7.60.1",
        "@opentelemetry/api": "^1.10.0",
        "@opentelemetry/api-logs": "^0.223.0",
        "@opentelemetry/instrumentation": "^0.223.1",
        "@opentelemetry/sdk-logs": "^0.223.1",
        "@opentelemetry/sdk-trace-base": "^2.12.1",
        "@opentelemetry/sdk-trace-node": "^2.12.1",
        "@opentelemetry/semantic-conventions": "^1.45.0",
        "@oxlint/plugins": "^1.93.0",
        "@playwright/browser-chromium": "1.65.1",
        "@playwright/browser-firefox": "1.65.1",
        "@playwright/browser-webkit": "1.65.1",
        "@types/domhandler": "^3.1.0",
        "@types/express": "^5.0.6",
        "@types/fs-extra": "^11.0.4",
        "@types/lodash.merge": "^4.6.9",
        "@types/mime-types": "^3.0.1",
        "@types/node": "^24.19.2",
        "@types/sax": "^1.2.7",
        "@types/semver": "^7.8.0",
        "@types/whatwg-mimetype": "^5.0.0",
        "@types/yargs": "^17.0.35",
        "@vitest/coverage-v8": "^5.1.0",
        "apify": "*",
        "apify-node-curl-impersonate": "^1.0.33",
        "basic-auth-parser": "^0.0.2",
        "body-parser": "^2.4.0",
        "camoufox-js": "^0.13.0",
        "crawlee": "workspace:*",
        "cross-env": "^10.1.0",
        "express": "^5.3.0",
        "fs-extra": "^11.4.1",
        "gen-esm-wrapper": "^1.1.3",
        "globby": "^15.0.0",
        "husky": "^9.1.7",
        "iconv-lite": "^0.7.3",
        "lerna": "^10.1.1",
        "lint-staged": "^17.6.0",
        "nock": "^14.0.17",
        "oxfmt": "^0.77.0",
        "oxlint": "^1.93.0",
        "oxlint-tsgolint": "^7.0.2006",
        "playwright": "catalog:",
        "portastic": "^1.0.1",
        "proxy": "^2.2.0",
        "puppeteer": "25.17.0",
        "rimraf": "^6.1.4",
        "tsx": "4.21.0",
        "turbo": "^2.11.11",
        "typescript": "^7.0.2",
        "typescript-v6": "npm:typescript@^6.0.3",
        "vitest": "^5.1.0",
        "zod": "catalog:"
    },
    "lint-staged": {
        "{packages,test,docs}/**/*.{js,ts,mjs,mts,cjs,cts}": [
            "oxlint --fix --no-error-on-unmatched-pattern",
            "oxfmt --write --no-error-on-unmatched-pattern"
        ]
    },
    "packageManager": "[email protected]",
    "devEngines": {
        "packageManager": {
            "name": "pnpm",
            "version": "12.21.0",
            "onFail": "error"
        }
    },
    "volta": {
        "node": "24.22.0",
        "pnpm": "12.21.0"
    }
}