Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: support TypeScript 4.8 (#1507)

Vlad Frangu4c3a5045931a
diff --git a/package-lock.json b/package-lock.json
index 48ebee650..1dcf96193 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,9 +14,9 @@
],
"devDependencies": {
"@apify/eslint-config-ts": "^0.2.3",
- "@apify/log": "^2.0.0",
+ "@apify/log": "^2.1.3",
"@apify/tsconfig": "^0.1.0",
- "@commitlint/config-conventional": "^17.0.2",
+ "@commitlint/config-conventional": "^17.1.0",
"@types/content-type": "^1.1.5",
"@types/deep-equal": "^1.0.1",
"@types/domhandler": "^2.4.2",
@@ -24,14 +24,14 @@
"@types/fs-extra": "^9.0.13",
"@types/htmlparser2": "^3.10.3",
"@types/inquirer": "^8.2.1",
- "@types/jest": "^28.1.1",
+ "@types/jest": "^28.1.8",
"@types/lodash.merge": "^4.6.7",
"@types/mime-types": "^2.1.1",
- "@types/node": "^18.0.3",
+ "@types/node": "^18.7.13",
"@types/proper-lockfile": "^4.1.2",
"@types/ps-tree": "^1.1.2",
"@types/rimraf": "^3.0.2",
- "@types/semver": "^7.3.9",
+ "@types/semver": "^7.3.12",
"@types/stream-json": "^1.7.2",
"@types/tough-cookie": "^4.0.2",
"@types/ws": "^7.4.7",
@@ -40,27 +40,27 @@
"apify": "3.0.2",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.20.0",
- "commitlint": "^17.0.2",
+ "commitlint": "^17.1.2",
"deep-equal": "^2.0.5",
- "eslint": "^8.17.0",
+ "eslint": "^8.23.0",
"express": "^4.18.1",
"fs-extra": "^10.1.0",
"gen-esm-wrapper": "^1.1.3",
- "globby": "^13.1.1",
+ "globby": "^13.1.2",
"husky": "^8.0.1",
- "jest": "^28.1.1",
- "lerna": "^5.1.1",
- "lint-staged": "^13.0.1",
- "node-gyp": "^9.0.0",
+ "jest": "^28.1.3",
+ "lerna": "^5.4.3",
+ "lint-staged": "^13.0.3",
+ "node-gyp": "^9.1.0",
"playwright": "1.25.1",
"portastic": "^1.0.1",
"proxy": "^1.0.2",
"puppeteer": "15.1.1",
- "ts-jest": "^28.0.4",
- "ts-node": "^10.8.1",
+ "ts-jest": "^28.0.8",
+ "ts-node": "^10.9.1",
"turbo": "1.4.3",
- "typescript": "^4.7.4",
- "ws": "^7.5.8"
+ "typescript": "^4.8.2",
+ "ws": "^7.5.9"
}
},
"node_modules/@ampproject/remapping": {
@@ -779,15 +779,15 @@
"dev": true
},
"node_modules/@commitlint/cli": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.0.3.tgz",
- "integrity": "sha512-oAo2vi5d8QZnAbtU5+0cR2j+A7PO8zuccux65R/EycwvsZrDVyW518FFrnJK2UQxbRtHFFIG+NjQ6vOiJV0Q8A==",
+ "version": "17.1.2",
+ "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.1.2.tgz",
+ "integrity": "sha512-h/4Hlka3bvCLbnxf0Er2ri5A44VMlbMSkdTRp8Adv2tRiklSTRIoPGs7OEXDv3EoDs2AAzILiPookgM4Gi7LOw==",
"dev": true,
"dependencies": {
"@commitlint/format": "^17.0.0",
- "@commitlint/lint": "^17.0.3",
- "@commitlint/load": "^17.0.3",
- "@commitlint/read": "^17.0.0",
+ "@commitlint/lint": "^17.1.0",
+ "@commitlint/load": "^17.1.2",
+ "@commitlint/read": "^17.1.0",
"@commitlint/types": "^17.0.0",
"execa": "^5.0.0",
"lodash": "^4.17.19",
@@ -803,9 +803,9 @@
}
},
"node_modules/@commitlint/config-conventional": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.0.3.tgz",
- "integrity": "sha512-HCnzTm5ATwwwzNVq5Y57poS0a1oOOcd5pc1MmBpLbGmSysc4i7F/++JuwtdFPu16sgM3H9J/j2zznRLOSGVO2A==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.1.0.tgz",
+ "integrity": "sha512-WU2p0c9/jLi8k2q2YrDV96Y8XVswQOceIQ/wyJvQxawJSCasLdRB3kUIYdNjOCJsxkpoUlV/b90ZPxp1MYZDiA==",
"dev": true,
"dependencies": {
"conventional-changelog-conventionalcommits": "^5.0.0"
@@ -815,9 +815,9 @@
}
},
"node_modules/@commitlint/config-validator": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.0.3.tgz",
- "integrity": "sha512-3tLRPQJKapksGE7Kee9axv+9z5I2GDHitDH4q63q7NmNA0wkB+DAorJ0RHz2/K00Zb1/MVdHzhCga34FJvDihQ==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.1.0.tgz",
+ "integrity": "sha512-Q1rRRSU09ngrTgeTXHq6ePJs2KrI+axPTgkNYDWSJIuS1Op4w3J30vUfSXjwn5YEJHklK3fSqWNHmBhmTR7Vdg==",
"dev": true,
"dependencies": {
"@commitlint/types": "^17.0.0",
@@ -863,9 +863,9 @@
}
},
"node_modules/@commitlint/is-ignored": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.0.3.tgz",
- "integrity": "sha512-/wgCXAvPtFTQZxsVxj7owLeRf5wwzcXLaYmrZPR4a87iD4sCvUIRl1/ogYrtOyUmHwWfQsvjqIB4mWE/SqWSnA==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.1.0.tgz",
+ "integrity": "sha512-JITWKDMHhIh8IpdIbcbuH9rEQJty1ZWelgjleTFrVRAcEwN/sPzk1aVUXRIZNXMJWbZj8vtXRJnFihrml8uECQ==",
"dev": true,
"dependencies": {
"@commitlint/types": "^17.0.0",
@@ -876,12 +876,12 @@
}
},
"node_modules/@commitlint/lint": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.0.3.tgz",
- "integrity": "sha512-2o1fk7JUdxBUgszyt41sHC/8Nd5PXNpkmuOo9jvGIjDHzOwXyV0PSdbEVTH3xGz9NEmjohFHr5l+N+T9fcxong==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.1.0.tgz",
+ "integrity": "sha512-ltpqM2ogt/+SDhUaScFo0MdscncEF96lvQTPMM/VTTWlw7sTGLLWkOOppsee2MN/uLNNWjQ7kqkd4h6JqoM9AQ==",
"dev": true,
"dependencies": {
- "@commitlint/is-ignored": "^17.0.3",
+ "@commitlint/is-ignored": "^17.1.0",
"@commitlint/parse": "^17.0.0",
"@commitlint/rules": "^17.0.0",
"@commitlint/types": "^17.0.0"
@@ -891,27 +891,34 @@
}
},
"node_modules/@commitlint/load": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.0.3.tgz",
- "integrity": "sha512-3Dhvr7GcKbKa/ey4QJ5MZH3+J7QFlARohUow6hftQyNjzoXXROm+RwpBes4dDFrXG1xDw9QPXA7uzrOShCd4bw==",
+ "version": "17.1.2",
+ "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.1.2.tgz",
+ "integrity": "sha512-sk2p/jFYAWLChIfOIp/MGSIn/WzZ0vkc3afw+l4X8hGEYkvDe4gQUUAVxjl/6xMRn0HgnSLMZ04xXh5pkTsmgg==",
"dev": true,
"dependencies": {
- "@commitlint/config-validator": "^17.0.3",
+ "@commitlint/config-validator": "^17.1.0",
"@commitlint/execute-rule": "^17.0.0",
- "@commitlint/resolve-extends": "^17.0.3",
+ "@commitlint/resolve-extends": "^17.1.0",
"@commitlint/types": "^17.0.0",
- "@types/node": ">=12",
+ "@types/node": "^14.0.0",
"chalk": "^4.1.0",
"cosmiconfig": "^7.0.0",
- "cosmiconfig-typescript-loader": "^2.0.0",
+ "cosmiconfig-typescript-loader": "^4.0.0",
"lodash": "^4.17.19",
"resolve-from": "^5.0.0",
+ "ts-node": "^10.8.1",
"typescript": "^4.6.4"
},
"engines": {
"node": ">=v14"
}
},
+ "node_modules/@commitlint/load/node_modules/@types/node": {
+ "version": "14.18.26",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.26.tgz",
+ "integrity": "sha512-0b+utRBSYj8L7XAp0d+DX7lI4cSmowNaaTkk6/1SKzbKkG+doLuPusB9EOvzLJ8ahJSk03bTLIL6cWaEd4dBKA==",
+ "dev": true
+ },
"node_modules/@commitlint/message": {
"version": "17.0.0",
"resolved": "https://registry.npmjs.org/@commitlint/message/-/message-17.0.0.tgz",
@@ -936,27 +943,28 @@
}
},
"node_modules/@commitlint/read": {
- "version": "17.0.0",
- "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.0.0.tgz",
- "integrity": "sha512-zkuOdZayKX3J6F6mPnVMzohK3OBrsEdOByIqp4zQjA9VLw1hMsDEFQ18rKgUc2adkZar+4S01QrFreDCfZgbxA==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.1.0.tgz",
+ "integrity": "sha512-73BoFNBA/3Ozo2JQvGsE0J8SdrJAWGfZQRSHqvKaqgmY042Su4gXQLqvAzgr55S9DI1l9TiU/5WDuh8IE86d/g==",
"dev": true,
"dependencies": {
"@commitlint/top-level": "^17.0.0",
"@commitlint/types": "^17.0.0",
"fs-extra": "^10.0.0",
- "git-raw-commits": "^2.0.0"
+ "git-raw-commits": "^2.0.0",
+ "minimist": "^1.2.6"
},
"engines": {
"node": ">=v14"
}
},
"node_modules/@commitlint/resolve-extends": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.0.3.tgz",
- "integrity": "sha512-H/RFMvrcBeJCMdnVC4i8I94108UDccIHrTke2tyQEg9nXQnR5/Hd6MhyNWkREvcrxh9Y+33JLb+PiPiaBxCtBA==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.1.0.tgz",
+ "integrity": "sha512-jqKm00LJ59T0O8O4bH4oMa4XyJVEOK4GzH8Qye9XKji+Q1FxhZznxMV/bDLyYkzbTodBt9sL0WLql8wMtRTbqQ==",
"dev": true,
"dependencies": {
- "@commitlint/config-validator": "^17.0.3",
+ "@commitlint/config-validator": "^17.1.0",
"@commitlint/types": "^17.0.0",
"import-fresh": "^3.0.0",
"lodash": "^4.17.19",
@@ -1095,14 +1103,14 @@
}
},
"node_modules/@eslint/eslintrc": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz",
- "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==",
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.1.tgz",
+ "integrity": "sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ==",
"dev": true,
"dependencies": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
- "espree": "^9.3.2",
+ "espree": "^9.4.0",
"globals": "^13.15.0",
"ignore": "^5.2.0",
"import-fresh": "^3.2.1",
@@ -1112,6 +1120,9 @@
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://opencollective.com/eslint"
}
},
"node_modules/@eslint/eslintrc/node_modules/ajv": {
@@ -1166,6 +1177,19 @@
"url": "https://github.com/sponsors/nzakas"
}
},
+ "node_modules/@humanwhocodes/module-importer": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
+ "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+ "dev": true,
+ "engines": {
+ "node": ">=12.22"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/nzakas"
+ }
+ },
"node_modules/@humanwhocodes/object-schema": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
@@ -5846,12 +5870,12 @@
}
},
"node_modules/commitlint": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.0.3.tgz",
- "integrity": "sha512-/KbIyrd6nmrRvu5zj8KKrjoC4z5V6hBmYphHgCFu75kPjHODg1XTtGFgbnb0AdSGBHlGMzmDvykO7ETs8wBKFg==",
+ "version": "17.1.2",
+ "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.1.2.tgz",
+ "integrity": "sha512-ykPXC3TUfYlL8tqz/VOeaJpTPNrcdKrR4Y7ShxjF6l8SlBN/+4YhBJpomG2dx8Ac43FM9OE5rdn1+h7NxsIcAQ==",
"dev": true,
"dependencies": {
- "@commitlint/cli": "^17.0.3",
+ "@commitlint/cli": "^17.1.2",
"@commitlint/types": "^17.0.0"
},
"bin": {
@@ -6164,14 +6188,10 @@
}
},
"node_modules/cosmiconfig-typescript-loader": {
- "version": "2.0.2",
- "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-2.0.2.tgz",
- "integrity": "sha512-KmE+bMjWMXJbkWCeY4FJX/npHuZPNr9XF9q9CIQ/bpFwi1qHfCmSiKarrCcRa0LO4fWjk93pVoeRtJAkTGcYNw==",
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.0.0.tgz",
+ "integrity": "sha512-cVpucSc2Tf+VPwCCR7SZzmQTQkPbkk4O01yXsYqXBIbjE1bhwqSyAgYQkRK1un4i0OPziTleqFhdkmOc4RQ/9g==",
"dev": true,
- "dependencies": {
- "cosmiconfig": "^7",
- "ts-node": "^10.8.1"
- },
"engines": {
"node": ">=12",
"npm": ">=6"
@@ -6179,6 +6199,7 @@
"peerDependencies": {
"@types/node": "*",
"cosmiconfig": ">=7",
+ "ts-node": ">=10",
"typescript": ">=3"
}
},
@@ -7077,14 +7098,15 @@
}
},
"node_modules/eslint": {
- "version": "8.22.0",
- "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz",
- "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==",
+ "version": "8.23.0",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.23.0.tgz",
+ "integrity": "sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA==",
"dev": true,
"dependencies": {
- "@eslint/eslintrc": "^1.3.0",
+ "@eslint/eslintrc": "^1.3.1",
"@humanwhocodes/config-array": "^0.10.4",
"@humanwhocodes/gitignore-to-minimatch": "^1.0.2",
+ "@humanwhocodes/module-importer": "^1.0.1",
"ajv": "^6.10.0",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.2",
@@ -7094,7 +7116,7 @@
"eslint-scope": "^7.1.1",
"eslint-utils": "^3.0.0",
"eslint-visitor-keys": "^3.3.0",
- "espree": "^9.3.3",
+ "espree": "^9.4.0",
"esquery": "^1.4.0",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
@@ -7119,8 +7141,7 @@
"regexpp": "^3.2.0",
"strip-ansi": "^6.0.1",
"strip-json-comments": "^3.1.0",
- "text-table": "^0.2.0",
- "v8-compile-cache": "^2.0.3"
+ "text-table": "^0.2.0"
},
"bin": {
"eslint": "bin/eslint.js"
@@ -7537,9 +7558,9 @@
}
},
"node_modules/espree": {
- "version": "9.3.3",
- "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz",
- "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==",
+ "version": "9.4.0",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz",
+ "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==",
"dev": true,
"dependencies": {
"acorn": "^8.8.0",
@@ -15567,9 +15588,9 @@
}
},
"node_modules/typescript": {
- "version": "4.7.4",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",
+ "version": "4.8.2",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",
+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
@@ -16576,7 +16597,8 @@
"got-scraping": "^3.2.9",
"iconv-lite": "^0.6.3",
"mime-types": "^2.1.35",
- "ow": "^0.28.1"
+ "ow": "^0.28.1",
+ "type-fest": "^2.19.0"
},
"engines": {
"node": ">=16.0.0"
@@ -16593,6 +16615,17 @@
"node": ">=0.10.0"
}
},
+ "packages/http-crawler/node_modules/type-fest": {
+ "version": "2.19.0",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz",
+ "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==",
+ "engines": {
+ "node": ">=12.20"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"packages/jsdom-crawler": {
"name": "@crawlee/jsdom",
"version": "3.0.4",
@@ -17562,15 +17595,15 @@
"dev": true
},
"@commitlint/cli": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.0.3.tgz",
- "integrity": "sha512-oAo2vi5d8QZnAbtU5+0cR2j+A7PO8zuccux65R/EycwvsZrDVyW518FFrnJK2UQxbRtHFFIG+NjQ6vOiJV0Q8A==",
+ "version": "17.1.2",
+ "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.1.2.tgz",
+ "integrity": "sha512-h/4Hlka3bvCLbnxf0Er2ri5A44VMlbMSkdTRp8Adv2tRiklSTRIoPGs7OEXDv3EoDs2AAzILiPookgM4Gi7LOw==",
"dev": true,
"requires": {
"@commitlint/format": "^17.0.0",
- "@commitlint/lint": "^17.0.3",
- "@commitlint/load": "^17.0.3",
- "@commitlint/read": "^17.0.0",
+ "@commitlint/lint": "^17.1.0",
+ "@commitlint/load": "^17.1.2",
+ "@commitlint/read": "^17.1.0",
"@commitlint/types": "^17.0.0",
"execa": "^5.0.0",
"lodash": "^4.17.19",
@@ -17580,18 +17613,18 @@
}
},
"@commitlint/config-conventional": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.0.3.tgz",
- "integrity": "sha512-HCnzTm5ATwwwzNVq5Y57poS0a1oOOcd5pc1MmBpLbGmSysc4i7F/++JuwtdFPu16sgM3H9J/j2zznRLOSGVO2A==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.1.0.tgz",
+ "integrity": "sha512-WU2p0c9/jLi8k2q2YrDV96Y8XVswQOceIQ/wyJvQxawJSCasLdRB3kUIYdNjOCJsxkpoUlV/b90ZPxp1MYZDiA==",
"dev": true,
"requires": {
"conventional-changelog-conventionalcommits": "^5.0.0"
}
},
"@commitlint/config-validator": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.0.3.tgz",
- "integrity": "sha512-3tLRPQJKapksGE7Kee9axv+9z5I2GDHitDH4q63q7NmNA0wkB+DAorJ0RHz2/K00Zb1/MVdHzhCga34FJvDihQ==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.1.0.tgz",
+ "integrity": "sha512-Q1rRRSU09ngrTgeTXHq6ePJs2KrI+axPTgkNYDWSJIuS1Op4w3J30vUfSXjwn5YEJHklK3fSqWNHmBhmTR7Vdg==",
"dev": true,
"requires": {
"@commitlint/types": "^17.0.0",
@@ -17625,9 +17658,9 @@
}
},
"@commitlint/is-ignored": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.0.3.tgz",
- "integrity": "sha512-/wgCXAvPtFTQZxsVxj7owLeRf5wwzcXLaYmrZPR4a87iD4sCvUIRl1/ogYrtOyUmHwWfQsvjqIB4mWE/SqWSnA==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.1.0.tgz",
+ "integrity": "sha512-JITWKDMHhIh8IpdIbcbuH9rEQJty1ZWelgjleTFrVRAcEwN/sPzk1aVUXRIZNXMJWbZj8vtXRJnFihrml8uECQ==",
"dev": true,
"requires": {
"@commitlint/types": "^17.0.0",
@@ -17635,34 +17668,43 @@
}
},
"@commitlint/lint": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.0.3.tgz",
- "integrity": "sha512-2o1fk7JUdxBUgszyt41sHC/8Nd5PXNpkmuOo9jvGIjDHzOwXyV0PSdbEVTH3xGz9NEmjohFHr5l+N+T9fcxong==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.1.0.tgz",
+ "integrity": "sha512-ltpqM2ogt/+SDhUaScFo0MdscncEF96lvQTPMM/VTTWlw7sTGLLWkOOppsee2MN/uLNNWjQ7kqkd4h6JqoM9AQ==",
"dev": true,
"requires": {
- "@commitlint/is-ignored": "^17.0.3",
+ "@commitlint/is-ignored": "^17.1.0",
"@commitlint/parse": "^17.0.0",
"@commitlint/rules": "^17.0.0",
"@commitlint/types": "^17.0.0"
}
},
"@commitlint/load": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.0.3.tgz",
- "integrity": "sha512-3Dhvr7GcKbKa/ey4QJ5MZH3+J7QFlARohUow6hftQyNjzoXXROm+RwpBes4dDFrXG1xDw9QPXA7uzrOShCd4bw==",
+ "version": "17.1.2",
+ "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.1.2.tgz",
+ "integrity": "sha512-sk2p/jFYAWLChIfOIp/MGSIn/WzZ0vkc3afw+l4X8hGEYkvDe4gQUUAVxjl/6xMRn0HgnSLMZ04xXh5pkTsmgg==",
"dev": true,
"requires": {
- "@commitlint/config-validator": "^17.0.3",
+ "@commitlint/config-validator": "^17.1.0",
"@commitlint/execute-rule": "^17.0.0",
- "@commitlint/resolve-extends": "^17.0.3",
+ "@commitlint/resolve-extends": "^17.1.0",
"@commitlint/types": "^17.0.0",
- "@types/node": ">=12",
+ "@types/node": "^14.0.0",
"chalk": "^4.1.0",
"cosmiconfig": "^7.0.0",
- "cosmiconfig-typescript-loader": "^2.0.0",
+ "cosmiconfig-typescript-loader": "^4.0.0",
"lodash": "^4.17.19",
"resolve-from": "^5.0.0",
+ "ts-node": "^10.8.1",
"typescript": "^4.6.4"
+ },
+ "dependencies": {
+ "@types/node": {
+ "version": "14.18.26",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.26.tgz",
+ "integrity": "sha512-0b+utRBSYj8L7XAp0d+DX7lI4cSmowNaaTkk6/1SKzbKkG+doLuPusB9EOvzLJ8ahJSk03bTLIL6cWaEd4dBKA==",
+ "dev": true
+ }
}
},
"@commitlint/message": {
@@ -17683,24 +17725,25 @@
}
},
"@commitlint/read": {
- "version": "17.0.0",
- "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.0.0.tgz",
- "integrity": "sha512-zkuOdZayKX3J6F6mPnVMzohK3OBrsEdOByIqp4zQjA9VLw1hMsDEFQ18rKgUc2adkZar+4S01QrFreDCfZgbxA==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.1.0.tgz",
+ "integrity": "sha512-73BoFNBA/3Ozo2JQvGsE0J8SdrJAWGfZQRSHqvKaqgmY042Su4gXQLqvAzgr55S9DI1l9TiU/5WDuh8IE86d/g==",
"dev": true,
"requires": {
"@commitlint/top-level": "^17.0.0",
"@commitlint/types": "^17.0.0",
"fs-extra": "^10.0.0",
- "git-raw-commits": "^2.0.0"
+ "git-raw-commits": "^2.0.0",
+ "minimist": "^1.2.6"
}
},
"@commitlint/resolve-extends": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.0.3.tgz",
- "integrity": "sha512-H/RFMvrcBeJCMdnVC4i8I94108UDccIHrTke2tyQEg9nXQnR5/Hd6MhyNWkREvcrxh9Y+33JLb+PiPiaBxCtBA==",
+ "version": "17.1.0",
+ "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.1.0.tgz",
+ "integrity": "sha512-jqKm00LJ59T0O8O4bH4oMa4XyJVEOK4GzH8Qye9XKji+Q1FxhZznxMV/bDLyYkzbTodBt9sL0WLql8wMtRTbqQ==",
"dev": true,
"requires": {
- "@commitlint/config-validator": "^17.0.3",
+ "@commitlint/config-validator": "^17.1.0",
"@commitlint/types": "^17.0.0",
"import-fresh": "^3.0.0",
"lodash": "^4.17.19",
@@ -17861,7 +17904,8 @@
"got-scraping": "^3.2.9",
"iconv-lite": "^0.6.3",
"mime-types": "^2.1.35",
- "ow": "^0.28.1"
+ "ow": "^0.28.1",
+ "type-fest": "*"
},
"dependencies": {
"iconv-lite": {
@@ -17871,6 +17915,11 @@
"requires": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
}
+ },
+ "type-fest": {
+ "version": "2.19.0",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz",
+ "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="
}
}
},
@@ -18144,14 +18193,14 @@
}
},
"@eslint/eslintrc": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz",
- "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==",
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.1.tgz",
+ "integrity": "sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ==",
"dev": true,
"requires": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
- "espree": "^9.3.2",
+ "espree": "^9.4.0",
"globals": "^13.15.0",
"ignore": "^5.2.0",
"import-fresh": "^3.2.1",
@@ -18203,6 +18252,12 @@
"integrity": "sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA==",
"dev": true
},
+ "@humanwhocodes/module-importer": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
+ "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+ "dev": true
+ },
"@humanwhocodes/object-schema": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
@@ -21933,12 +21988,12 @@
"dev": true
},
"commitlint": {
- "version": "17.0.3",
- "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.0.3.tgz",
- "integrity": "sha512-/KbIyrd6nmrRvu5zj8KKrjoC4z5V6hBmYphHgCFu75kPjHODg1XTtGFgbnb0AdSGBHlGMzmDvykO7ETs8wBKFg==",
+ "version": "17.1.2",
+ "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.1.2.tgz",
+ "integrity": "sha512-ykPXC3TUfYlL8tqz/VOeaJpTPNrcdKrR4Y7ShxjF6l8SlBN/+4YhBJpomG2dx8Ac43FM9OE5rdn1+h7NxsIcAQ==",
"dev": true,
"requires": {
- "@commitlint/cli": "^17.0.3",
+ "@commitlint/cli": "^17.1.2",
"@commitlint/types": "^17.0.0"
}
},
@@ -22190,14 +22245,11 @@
}
},
"cosmiconfig-typescript-loader": {
- "version": "2.0.2",
- "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-2.0.2.tgz",
- "integrity": "sha512-KmE+bMjWMXJbkWCeY4FJX/npHuZPNr9XF9q9CIQ/bpFwi1qHfCmSiKarrCcRa0LO4fWjk93pVoeRtJAkTGcYNw==",
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.0.0.tgz",
+ "integrity": "sha512-cVpucSc2Tf+VPwCCR7SZzmQTQkPbkk4O01yXsYqXBIbjE1bhwqSyAgYQkRK1un4i0OPziTleqFhdkmOc4RQ/9g==",
"dev": true,
- "requires": {
- "cosmiconfig": "^7",
- "ts-node": "^10.8.1"
- }
+ "requires": {}
},
"crawlee": {
"version": "file:packages/crawlee",
@@ -22890,14 +22942,15 @@
}
},
"eslint": {
- "version": "8.22.0",
- "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz",
- "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==",
+ "version": "8.23.0",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.23.0.tgz",
+ "integrity": "sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA==",
"dev": true,
"requires": {
- "@eslint/eslintrc": "^1.3.0",
+ "@eslint/eslintrc": "^1.3.1",
"@humanwhocodes/config-array": "^0.10.4",
"@humanwhocodes/gitignore-to-minimatch": "^1.0.2",
+ "@humanwhocodes/module-importer": "^1.0.1",
"ajv": "^6.10.0",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.2",
@@ -22907,7 +22960,7 @@
"eslint-scope": "^7.1.1",
"eslint-utils": "^3.0.0",
"eslint-visitor-keys": "^3.3.0",
- "espree": "^9.3.3",
+ "espree": "^9.4.0",
"esquery": "^1.4.0",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
@@ -22932,8 +22985,7 @@
"regexpp": "^3.2.0",
"strip-ansi": "^6.0.1",
"strip-json-comments": "^3.1.0",
- "text-table": "^0.2.0",
- "v8-compile-cache": "^2.0.3"
+ "text-table": "^0.2.0"
},
"dependencies": {
"ajv": {
@@ -23246,9 +23298,9 @@
"dev": true
},
"espree": {
- "version": "9.3.3",
- "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz",
- "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==",
+ "version": "9.4.0",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz",
+ "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==",
"dev": true,
"requires": {
"acorn": "^8.8.0",
@@ -29237,9 +29289,9 @@
}
},
"typescript": {
- "version": "4.7.4",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",
+ "version": "4.8.2",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",
+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==",
"dev": true
},
"uglify-js": {
diff --git a/package.json b/package.json
index 25065af48..0f51fccd0 100644
--- a/package.json
+++ b/package.json
@@ -52,9 +52,9 @@
},
"devDependencies": {
"@apify/eslint-config-ts": "^0.2.3",
- "@apify/log": "^2.0.0",
+ "@apify/log": "^2.1.3",
"@apify/tsconfig": "^0.1.0",
- "@commitlint/config-conventional": "^17.0.2",
+ "@commitlint/config-conventional": "^17.1.0",
"@types/content-type": "^1.1.5",
"@types/deep-equal": "^1.0.1",
"@types/domhandler": "^2.4.2",
@@ -62,14 +62,14 @@
"@types/fs-extra": "^9.0.13",
"@types/htmlparser2": "^3.10.3",
"@types/inquirer": "^8.2.1",
- "@types/jest": "^28.1.1",
+ "@types/jest": "^28.1.8",
"@types/lodash.merge": "^4.6.7",
"@types/mime-types": "^2.1.1",
- "@types/node": "^18.0.3",
+ "@types/node": "^18.7.13",
"@types/proper-lockfile": "^4.1.2",
"@types/ps-tree": "^1.1.2",
"@types/rimraf": "^3.0.2",
- "@types/semver": "^7.3.9",
+ "@types/semver": "^7.3.12",
"@types/stream-json": "^1.7.2",
"@types/tough-cookie": "^4.0.2",
"@types/ws": "^7.4.7",
@@ -78,27 +78,27 @@
"apify": "3.0.2",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.20.0",
- "commitlint": "^17.0.2",
+ "commitlint": "^17.1.2",
"deep-equal": "^2.0.5",
- "eslint": "^8.17.0",
+ "eslint": "^8.23.0",
"express": "^4.18.1",
"fs-extra": "^10.1.0",
"gen-esm-wrapper": "^1.1.3",
- "globby": "^13.1.1",
+ "globby": "^13.1.2",
"husky": "^8.0.1",
- "jest": "^28.1.1",
- "lerna": "^5.1.1",
- "lint-staged": "^13.0.1",
- "node-gyp": "^9.0.0",
+ "jest": "^28.1.3",
+ "lerna": "^5.4.3",
+ "lint-staged": "^13.0.3",
+ "node-gyp": "^9.1.0",
"playwright": "1.25.1",
"portastic": "^1.0.1",
"proxy": "^1.0.2",
"puppeteer": "15.1.1",
- "ts-jest": "^28.0.4",
- "ts-node": "^10.8.1",
+ "ts-jest": "^28.0.8",
+ "ts-node": "^10.9.1",
"turbo": "1.4.3",
- "typescript": "^4.7.4",
- "ws": "^7.5.8"
+ "typescript": "^4.8.2",
+ "ws": "^7.5.9"
},
"packageManager": "[email protected]"
}
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 3473d336f..7e6cc2fbb 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -55,7 +55,7 @@ export interface BrowserCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOpti
export type BrowserHook<
Context = BrowserCrawlingContext,
- GoToOptions extends Record<PropertyKey, any> | undefined = Dictionary
+ GoToOptions extends Dictionary | undefined = Dictionary
> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
export interface BrowserCrawlerOptions<
@@ -285,9 +285,9 @@ export interface BrowserCrawlerOptions<
*/
export abstract class BrowserCrawler<
InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
- LaunchOptions = Dictionary,
+ LaunchOptions extends Dictionary | undefined = Dictionary,
Context extends BrowserCrawlingContext = BrowserCrawlingContext,
- GoToOptions extends Record<PropertyKey, any> = Dictionary
+ GoToOptions extends Dictionary = Dictionary
> extends BasicCrawler<Context> {
/**
* A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.
diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.ts
index 8cc2aed9e..016fc815e 100644
--- a/packages/browser-crawler/src/internals/browser-launcher.ts
+++ b/packages/browser-crawler/src/internals/browser-launcher.ts
@@ -69,7 +69,7 @@ export abstract class BrowserLauncher<
Plugin extends BrowserPlugin,
Launcher = Plugin['library'],
T extends Constructor<Plugin> = Constructor<Plugin>,
- LaunchOptions = Partial<Parameters<Plugin['launch']>[0]>,
+ LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>,
LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>,
> {
launcher: Launcher;
diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.ts
index 2a52c29cd..42f8e4020 100644
--- a/packages/browser-pool/src/abstract-classes/browser-controller.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts
@@ -1,7 +1,7 @@
import { nanoid } from 'nanoid';
import { TypedEmitter } from 'tiny-typed-emitter';
import { tryCancel } from '@apify/timeout';
-import type { Cookie } from '@crawlee/types';
+import type { Cookie, Dictionary } from '@crawlee/types';
import { BROWSER_CONTROLLER_EVENTS } from '../events';
import type { LaunchContext } from '../launch-context';
import { log } from '../logger';
@@ -13,7 +13,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
export interface BrowserControllerEvents<
Library extends CommonLibrary,
- LibraryOptions = Parameters<Library['launch']>[0],
+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
@@ -32,7 +32,7 @@ export interface BrowserControllerEvents<
*/
export abstract class BrowserController<
Library extends CommonLibrary = CommonLibrary,
- LibraryOptions = Parameters<Library['launch']>[0],
+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
index 4eedb5c5a..e18fd6a2a 100644
--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
@@ -1,4 +1,5 @@
import merge from 'lodash.merge';
+import type { Dictionary } from '@crawlee/types';
import type { LaunchContextOptions } from '../launch-context';
import { LaunchContext } from '../launch-context';
import type { BrowserController } from './browser-controller';
@@ -24,7 +25,7 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/5
*/
export interface CommonLibrary {
product?: string;
- launch(...args: unknown[]): Promise<CommonBrowser>;
+ launch(opts?: Dictionary): Promise<CommonBrowser>;
name?: () => string;
}
@@ -74,7 +75,7 @@ export interface BrowserPluginOptions<LibraryOptions> {
export interface CreateLaunchContextOptions<
Library extends CommonLibrary,
- LibraryOptions = Parameters<Library['launch']>[0],
+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
@@ -88,7 +89,7 @@ export interface CreateLaunchContextOptions<
*/
export abstract class BrowserPlugin<
Library extends CommonLibrary = CommonLibrary,
- LibraryOptions = Parameters<Library['launch']>[0],
+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
@@ -163,7 +164,9 @@ export abstract class BrowserPlugin<
async launch(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = this.createLaunchContext(),
): Promise<LaunchResult> {
- const { proxyUrl, launchOptions }: { proxyUrl?: string; launchOptions: Record<string, any> } = launchContext;
+ launchContext.launchOptions ??= {} as LibraryOptions;
+
+ const { proxyUrl, launchOptions } = launchContext;
if (proxyUrl) {
await this._addProxyToLaunchOptions(launchContext);
@@ -171,20 +174,20 @@ export abstract class BrowserPlugin<
if (this._isChromiumBasedBrowser(launchContext)) {
// This will set the args for chromium based browsers to hide the webdriver.
- launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
+ (launchOptions as Dictionary).args = this._mergeArgsToHideWebdriver(launchOptions!.args);
// When User-Agent is not set, and we're using Chromium in headless mode,
// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
// as otherwise 'HeadlessChrome' is present in User-Agent string.
- const userAgent = launchOptions.args.find((arg: string) => arg.startsWith('--user-agent'));
- if (launchOptions.headless && !launchContext.fingerprint && !userAgent) {
- launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
+ const userAgent = launchOptions!.args.find((arg: string) => arg.startsWith('--user-agent'));
+ if (launchOptions!.headless && !launchContext.fingerprint && !userAgent) {
+ launchOptions!.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
}
}
return this._launch(launchContext);
}
- private _mergeArgsToHideWebdriver(originalArgs: string[]): string[] {
+ private _mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {
if (!originalArgs?.length) {
return ['--disable-blink-features=AutomationControlled'];
}
diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.ts
index 35dfd21ea..8988066a9 100644
--- a/packages/browser-pool/src/launch-context.ts
+++ b/packages/browser-pool/src/launch-context.ts
@@ -1,3 +1,4 @@
+import type { Dictionary } from '@crawlee/types';
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './abstract-classes/browser-plugin';
import type { UnwrapPromise } from './utils';
@@ -11,7 +12,7 @@ import type { UnwrapPromise } from './utils';
*/
export interface LaunchContextOptions<
Library extends CommonLibrary = CommonLibrary,
- LibraryOptions = Parameters<Library['launch']>[0],
+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
@@ -52,7 +53,7 @@ export interface LaunchContextOptions<
export class LaunchContext<
Library extends CommonLibrary = CommonLibrary,
- LibraryOptions = Parameters<Library['launch']>[0],
+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index a2fbe07d0..c9748ec5e 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -1,5 +1,4 @@
import type { Dictionary } from '@crawlee/types';
-// eslint isn't compatible with `import type`
import type Puppeteer from './puppeteer-proxy-per-page';
import type { Browser, Target, BrowserContext } from './puppeteer-proxy-per-page';
import type { BrowserController } from '../abstract-classes/browser-controller';
diff --git a/packages/cli/src/commands/CreateProjectCommand.ts b/packages/cli/src/commands/CreateProjectCommand.ts
index 0582278ec..f4c626616 100644
--- a/packages/cli/src/commands/CreateProjectCommand.ts
+++ b/packages/cli/src/commands/CreateProjectCommand.ts
@@ -33,7 +33,7 @@ function rewrite(path: string, replacer: (from: string) => string): void {
export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectArgs> {
command = 'create [project-name]';
describe = 'Creates a new Crawlee project directory from a selected boilerplate template.';
- builder = async (args: Argv) => {
+ builder = async (args: Argv<T>) => {
const manifest = await fetchManifest();
const choices = manifest.templates.map((t) => t.name);
diff --git a/packages/cli/src/commands/RunProjectCommand.ts b/packages/cli/src/commands/RunProjectCommand.ts
index 9afc834ec..e69f66d94 100644
--- a/packages/cli/src/commands/RunProjectCommand.ts
+++ b/packages/cli/src/commands/RunProjectCommand.ts
@@ -9,7 +9,7 @@ interface RunProjectArgs {
export class RunProjectCommand<T> implements CommandModule<T, RunProjectArgs> {
command = 'run';
describe = 'Run crawlee project';
- builder = async (args: Argv) => {
+ builder = async (args: Argv<T>) => {
args.option('purge', {
alias: 't',
default: true,
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index 90b37de37..81153a149 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -104,7 +104,7 @@ export class KeyValueStore {
private persistStateEventStarted = false;
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
- private readonly cache = new Map<string, unknown>();
+ private readonly cache = new Map<string, Dictionary>();
/**
* @internal
@@ -160,7 +160,7 @@ export class KeyValueStore {
}
const value = await this.getValue<T>(key, defaultValue);
- this.cache.set(key, value);
+ this.cache.set(key, value!);
this.ensurePersistStateEvent();
return value!;
@@ -241,9 +241,9 @@ export class KeyValueStore {
const optionsCopy = { ...options };
// If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
- const cachedValue = this.cache.get(key) as T;
+ const cachedValue = this.cache.get(key);
- if (this.cache.has(key) && cachedValue !== value) {
+ if (cachedValue && cachedValue !== value) {
if (value === null) {
// Cached state can be only object, so a propagation of `null` means removing all its properties.
Object.keys(cachedValue).forEach((k) => this.cache.delete(k));
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index 220bad699..5727f03c1 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -62,6 +62,7 @@
"got-scraping": "^3.2.9",
"iconv-lite": "^0.6.3",
"mime-types": "^2.1.35",
- "ow": "^0.28.1"
+ "ow": "^0.28.1",
+ "type-fest": "^2.19.0"
}
}
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 28777bcbc..b85ca5e19 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -25,7 +25,8 @@ import contentTypeParser from 'content-type';
import mime from 'mime-types';
import type { OptionsInit, Method, Request as GotRequest, Response as GotResponse, GotOptionsInit } from 'got-scraping';
import { gotScraping, TimeoutError } from 'got-scraping';
-import { extname } from 'path';
+import type { JsonValue } from 'type-fest';
+import { extname } from 'node:path';
import type { IncomingHttpHeaders, IncomingMessage } from 'node:http';
import iconv from 'iconv-lite';
import ow from 'ow';
@@ -49,7 +50,7 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS: AutoscaledPoolOptions = {
export type HttpErrorHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext> extends BasicCrawlerOptions<Context> {
@@ -155,7 +156,7 @@ export type InternalHttpHook<Context> = (
export type HttpHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
/**
@@ -163,7 +164,7 @@ export type HttpHook<
*/
export interface InternalHttpCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
Crawler = HttpCrawler<any>
> extends CrawlingContext<UserData> {
/**
@@ -188,12 +189,12 @@ export interface InternalHttpCrawlingContext<
sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
}
-export interface HttpCrawlingContext<UserData = any, JSONData = any>
+export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any>
extends InternalHttpCrawlingContext<UserData, JSONData, HttpCrawler<HttpCrawlingContext<UserData, JSONData>>> {}
export type HttpRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
/**
diff --git a/website/package-lock.json b/website/package-lock.json
index 384d5d065..a1675d8f8 100644
--- a/website/package-lock.json
+++ b/website/package-lock.json
@@ -32,7 +32,8 @@
"got": "^11.8.5",
"path-browserify": "^1.0.1",
"prettier": "^2.6.2",
- "rimraf": "^3.0.2"
+ "rimraf": "^3.0.2",
+ "typescript": "^4.8.2"
}
},
"node_modules/@algolia/autocomplete-core": {
@@ -12471,13 +12472,13 @@
}
},
"node_modules/shiki": {
- "version": "0.10.1",
- "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.10.1.tgz",
- "integrity": "sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==",
+ "version": "0.11.1",
+ "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.11.1.tgz",
+ "integrity": "sha512-EugY9VASFuDqOexOgXR18ZV+TbFrQHeCpEYaXamO+SZlsnT/2LxuLBX25GGtIrwaEVFXUAbUQ601SWE2rMwWHA==",
"dependencies": {
"jsonc-parser": "^3.0.0",
"vscode-oniguruma": "^1.6.1",
- "vscode-textmate": "5.2.0"
+ "vscode-textmate": "^6.0.0"
}
},
"node_modules/side-channel": {
@@ -13260,14 +13261,14 @@
}
},
"node_modules/typedoc": {
- "version": "0.23.10",
- "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.10.tgz",
- "integrity": "sha512-03EUiu/ZuScUBMnY6p0lY+HTH8SwhzvRE3gImoemdPDWXPXlks83UGTx++lyquWeB1MTwm9D9Ca8RIjkK3AFfQ==",
+ "version": "0.23.11",
+ "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.11.tgz",
+ "integrity": "sha512-FhZ2HfqlS++53UwHk4txCsTrTlpYR0So/0osMyBeP1E7llRNRqycJGfYK1qx9Wvvv5VO8tGdpwzOwDW5FrTi7A==",
"dependencies": {
"lunr": "^2.3.9",
- "marked": "^4.0.18",
+ "marked": "^4.0.19",
"minimatch": "^5.1.0",
- "shiki": "^0.10.1"
+ "shiki": "^0.11.1"
},
"bin": {
"typedoc": "bin/typedoc"
@@ -13276,7 +13277,7 @@
"node": ">= 14.14"
},
"peerDependencies": {
- "typescript": "4.6.x || 4.7.x"
+ "typescript": "4.6.x || 4.7.x || 4.8.x"
}
},
"node_modules/typedoc/node_modules/brace-expansion": {
@@ -13299,10 +13300,9 @@
}
},
"node_modules/typescript": {
- "version": "4.7.4",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",
- "peer": true,
+ "version": "4.8.2",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",
+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -13943,9 +13943,9 @@
"integrity": "sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA=="
},
"node_modules/vscode-textmate": {
- "version": "5.2.0",
- "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz",
- "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ=="
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-6.0.0.tgz",
+ "integrity": "sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ=="
},
"node_modules/wait-on": {
"version": "6.0.1",
@@ -23618,13 +23618,13 @@
}
},
"shiki": {
- "version": "0.10.1",
- "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.10.1.tgz",
- "integrity": "sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==",
+ "version": "0.11.1",
+ "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.11.1.tgz",
+ "integrity": "sha512-EugY9VASFuDqOexOgXR18ZV+TbFrQHeCpEYaXamO+SZlsnT/2LxuLBX25GGtIrwaEVFXUAbUQ601SWE2rMwWHA==",
"requires": {
"jsonc-parser": "^3.0.0",
"vscode-oniguruma": "^1.6.1",
- "vscode-textmate": "5.2.0"
+ "vscode-textmate": "^6.0.0"
}
},
"side-channel": {
@@ -24194,14 +24194,14 @@
}
},
"typedoc": {
- "version": "0.23.10",
- "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.10.tgz",
- "integrity": "sha512-03EUiu/ZuScUBMnY6p0lY+HTH8SwhzvRE3gImoemdPDWXPXlks83UGTx++lyquWeB1MTwm9D9Ca8RIjkK3AFfQ==",
+ "version": "0.23.11",
+ "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.11.tgz",
+ "integrity": "sha512-FhZ2HfqlS++53UwHk4txCsTrTlpYR0So/0osMyBeP1E7llRNRqycJGfYK1qx9Wvvv5VO8tGdpwzOwDW5FrTi7A==",
"requires": {
"lunr": "^2.3.9",
- "marked": "^4.0.18",
+ "marked": "^4.0.19",
"minimatch": "^5.1.0",
- "shiki": "^0.10.1"
+ "shiki": "^0.11.1"
},
"dependencies": {
"brace-expansion": {
@@ -24223,10 +24223,9 @@
}
},
"typescript": {
- "version": "4.7.4",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",
- "peer": true
+ "version": "4.8.2",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",
+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw=="
},
"ua-parser-js": {
"version": "0.7.31",
@@ -24647,9 +24646,9 @@
"integrity": "sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA=="
},
"vscode-textmate": {
- "version": "5.2.0",
- "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz",
- "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ=="
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-6.0.0.tgz",
+ "integrity": "sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ=="
},
"wait-on": {
"version": "6.0.1",
diff --git a/website/package.json b/website/package.json
index 1f26e8d02..ebd3afd32 100644
--- a/website/package.json
+++ b/website/package.json
@@ -25,7 +25,8 @@
"got": "^11.8.5",
"path-browserify": "^1.0.1",
"prettier": "^2.6.2",
- "rimraf": "^3.0.2"
+ "rimraf": "^3.0.2",
+ "typescript": "^4.8.2"
},
"dependencies": {
"@docusaurus/core": "2.0.1",