Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: support TypeScript 4.8 (#1507)
Vlad Frangu4c3a5045931a
diff --git a/package-lock.json b/package-lock.jsonindex 48ebee650..1dcf96193 100644--- a/package-lock.json+++ b/package-lock.json@@ -14,9 +14,9 @@],"devDependencies": {"@apify/eslint-config-ts": "^0.2.3",- "@apify/log": "^2.0.0",+ "@apify/log": "^2.1.3","@apify/tsconfig": "^0.1.0",- "@commitlint/config-conventional": "^17.0.2",+ "@commitlint/config-conventional": "^17.1.0","@types/content-type": "^1.1.5","@types/deep-equal": "^1.0.1","@types/domhandler": "^2.4.2",@@ -24,14 +24,14 @@"@types/fs-extra": "^9.0.13","@types/htmlparser2": "^3.10.3","@types/inquirer": "^8.2.1",- "@types/jest": "^28.1.1",+ "@types/jest": "^28.1.8","@types/lodash.merge": "^4.6.7","@types/mime-types": "^2.1.1",- "@types/node": "^18.0.3",+ "@types/node": "^18.7.13","@types/proper-lockfile": "^4.1.2","@types/ps-tree": "^1.1.2","@types/rimraf": "^3.0.2",- "@types/semver": "^7.3.9",+ "@types/semver": "^7.3.12","@types/stream-json": "^1.7.2","@types/tough-cookie": "^4.0.2","@types/ws": "^7.4.7",@@ -40,27 +40,27 @@"apify": "3.0.2","basic-auth-parser": "^0.0.2","body-parser": "^1.20.0",- "commitlint": "^17.0.2",+ "commitlint": "^17.1.2","deep-equal": "^2.0.5",- "eslint": "^8.17.0",+ "eslint": "^8.23.0","express": "^4.18.1","fs-extra": "^10.1.0","gen-esm-wrapper": "^1.1.3",- "globby": "^13.1.1",+ "globby": "^13.1.2","husky": "^8.0.1",- "jest": "^28.1.1",- "lerna": "^5.1.1",- "lint-staged": "^13.0.1",- "node-gyp": "^9.0.0",+ "jest": "^28.1.3",+ "lerna": "^5.4.3",+ "lint-staged": "^13.0.3",+ "node-gyp": "^9.1.0","playwright": "1.25.1","portastic": "^1.0.1","proxy": "^1.0.2","puppeteer": "15.1.1",- "ts-jest": "^28.0.4",- "ts-node": "^10.8.1",+ "ts-jest": "^28.0.8",+ "ts-node": "^10.9.1","turbo": "1.4.3",- "typescript": "^4.7.4",- "ws": "^7.5.8"+ "typescript": "^4.8.2",+ "ws": "^7.5.9"}},"node_modules/@ampproject/remapping": {@@ -779,15 +779,15 @@"dev": true},"node_modules/@commitlint/cli": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.0.3.tgz",- "integrity": "sha512-oAo2vi5d8QZnAbtU5+0cR2j+A7PO8zuccux65R/EycwvsZrDVyW518FFrnJK2UQxbRtHFFIG+NjQ6vOiJV0Q8A==",+ "version": "17.1.2",+ "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.1.2.tgz",+ "integrity": "sha512-h/4Hlka3bvCLbnxf0Er2ri5A44VMlbMSkdTRp8Adv2tRiklSTRIoPGs7OEXDv3EoDs2AAzILiPookgM4Gi7LOw==","dev": true,"dependencies": {"@commitlint/format": "^17.0.0",- "@commitlint/lint": "^17.0.3",- "@commitlint/load": "^17.0.3",- "@commitlint/read": "^17.0.0",+ "@commitlint/lint": "^17.1.0",+ "@commitlint/load": "^17.1.2",+ "@commitlint/read": "^17.1.0","@commitlint/types": "^17.0.0","execa": "^5.0.0","lodash": "^4.17.19",@@ -803,9 +803,9 @@}},"node_modules/@commitlint/config-conventional": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.0.3.tgz",- "integrity": "sha512-HCnzTm5ATwwwzNVq5Y57poS0a1oOOcd5pc1MmBpLbGmSysc4i7F/++JuwtdFPu16sgM3H9J/j2zznRLOSGVO2A==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.1.0.tgz",+ "integrity": "sha512-WU2p0c9/jLi8k2q2YrDV96Y8XVswQOceIQ/wyJvQxawJSCasLdRB3kUIYdNjOCJsxkpoUlV/b90ZPxp1MYZDiA==","dev": true,"dependencies": {"conventional-changelog-conventionalcommits": "^5.0.0"@@ -815,9 +815,9 @@}},"node_modules/@commitlint/config-validator": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.0.3.tgz",- "integrity": "sha512-3tLRPQJKapksGE7Kee9axv+9z5I2GDHitDH4q63q7NmNA0wkB+DAorJ0RHz2/K00Zb1/MVdHzhCga34FJvDihQ==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.1.0.tgz",+ "integrity": "sha512-Q1rRRSU09ngrTgeTXHq6ePJs2KrI+axPTgkNYDWSJIuS1Op4w3J30vUfSXjwn5YEJHklK3fSqWNHmBhmTR7Vdg==","dev": true,"dependencies": {"@commitlint/types": "^17.0.0",@@ -863,9 +863,9 @@}},"node_modules/@commitlint/is-ignored": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.0.3.tgz",- "integrity": "sha512-/wgCXAvPtFTQZxsVxj7owLeRf5wwzcXLaYmrZPR4a87iD4sCvUIRl1/ogYrtOyUmHwWfQsvjqIB4mWE/SqWSnA==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.1.0.tgz",+ "integrity": "sha512-JITWKDMHhIh8IpdIbcbuH9rEQJty1ZWelgjleTFrVRAcEwN/sPzk1aVUXRIZNXMJWbZj8vtXRJnFihrml8uECQ==","dev": true,"dependencies": {"@commitlint/types": "^17.0.0",@@ -876,12 +876,12 @@}},"node_modules/@commitlint/lint": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.0.3.tgz",- "integrity": "sha512-2o1fk7JUdxBUgszyt41sHC/8Nd5PXNpkmuOo9jvGIjDHzOwXyV0PSdbEVTH3xGz9NEmjohFHr5l+N+T9fcxong==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.1.0.tgz",+ "integrity": "sha512-ltpqM2ogt/+SDhUaScFo0MdscncEF96lvQTPMM/VTTWlw7sTGLLWkOOppsee2MN/uLNNWjQ7kqkd4h6JqoM9AQ==","dev": true,"dependencies": {- "@commitlint/is-ignored": "^17.0.3",+ "@commitlint/is-ignored": "^17.1.0","@commitlint/parse": "^17.0.0","@commitlint/rules": "^17.0.0","@commitlint/types": "^17.0.0"@@ -891,27 +891,34 @@}},"node_modules/@commitlint/load": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.0.3.tgz",- "integrity": "sha512-3Dhvr7GcKbKa/ey4QJ5MZH3+J7QFlARohUow6hftQyNjzoXXROm+RwpBes4dDFrXG1xDw9QPXA7uzrOShCd4bw==",+ "version": "17.1.2",+ "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.1.2.tgz",+ "integrity": "sha512-sk2p/jFYAWLChIfOIp/MGSIn/WzZ0vkc3afw+l4X8hGEYkvDe4gQUUAVxjl/6xMRn0HgnSLMZ04xXh5pkTsmgg==","dev": true,"dependencies": {- "@commitlint/config-validator": "^17.0.3",+ "@commitlint/config-validator": "^17.1.0","@commitlint/execute-rule": "^17.0.0",- "@commitlint/resolve-extends": "^17.0.3",+ "@commitlint/resolve-extends": "^17.1.0","@commitlint/types": "^17.0.0",- "@types/node": ">=12",+ "@types/node": "^14.0.0","chalk": "^4.1.0","cosmiconfig": "^7.0.0",- "cosmiconfig-typescript-loader": "^2.0.0",+ "cosmiconfig-typescript-loader": "^4.0.0","lodash": "^4.17.19","resolve-from": "^5.0.0",+ "ts-node": "^10.8.1","typescript": "^4.6.4"},"engines": {"node": ">=v14"}},+ "node_modules/@commitlint/load/node_modules/@types/node": {+ "version": "14.18.26",+ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.26.tgz",+ "integrity": "sha512-0b+utRBSYj8L7XAp0d+DX7lI4cSmowNaaTkk6/1SKzbKkG+doLuPusB9EOvzLJ8ahJSk03bTLIL6cWaEd4dBKA==",+ "dev": true+ },"node_modules/@commitlint/message": {"version": "17.0.0","resolved": "https://registry.npmjs.org/@commitlint/message/-/message-17.0.0.tgz",@@ -936,27 +943,28 @@}},"node_modules/@commitlint/read": {- "version": "17.0.0",- "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.0.0.tgz",- "integrity": "sha512-zkuOdZayKX3J6F6mPnVMzohK3OBrsEdOByIqp4zQjA9VLw1hMsDEFQ18rKgUc2adkZar+4S01QrFreDCfZgbxA==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.1.0.tgz",+ "integrity": "sha512-73BoFNBA/3Ozo2JQvGsE0J8SdrJAWGfZQRSHqvKaqgmY042Su4gXQLqvAzgr55S9DI1l9TiU/5WDuh8IE86d/g==","dev": true,"dependencies": {"@commitlint/top-level": "^17.0.0","@commitlint/types": "^17.0.0","fs-extra": "^10.0.0",- "git-raw-commits": "^2.0.0"+ "git-raw-commits": "^2.0.0",+ "minimist": "^1.2.6"},"engines": {"node": ">=v14"}},"node_modules/@commitlint/resolve-extends": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.0.3.tgz",- "integrity": "sha512-H/RFMvrcBeJCMdnVC4i8I94108UDccIHrTke2tyQEg9nXQnR5/Hd6MhyNWkREvcrxh9Y+33JLb+PiPiaBxCtBA==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.1.0.tgz",+ "integrity": "sha512-jqKm00LJ59T0O8O4bH4oMa4XyJVEOK4GzH8Qye9XKji+Q1FxhZznxMV/bDLyYkzbTodBt9sL0WLql8wMtRTbqQ==","dev": true,"dependencies": {- "@commitlint/config-validator": "^17.0.3",+ "@commitlint/config-validator": "^17.1.0","@commitlint/types": "^17.0.0","import-fresh": "^3.0.0","lodash": "^4.17.19",@@ -1095,14 +1103,14 @@}},"node_modules/@eslint/eslintrc": {- "version": "1.3.0",- "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz",- "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==",+ "version": "1.3.1",+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.1.tgz",+ "integrity": "sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ==","dev": true,"dependencies": {"ajv": "^6.12.4","debug": "^4.3.2",- "espree": "^9.3.2",+ "espree": "^9.4.0","globals": "^13.15.0","ignore": "^5.2.0","import-fresh": "^3.2.1",@@ -1112,6 +1120,9 @@},"engines": {"node": "^12.22.0 || ^14.17.0 || >=16.0.0"+ },+ "funding": {+ "url": "https://opencollective.com/eslint"}},"node_modules/@eslint/eslintrc/node_modules/ajv": {@@ -1166,6 +1177,19 @@"url": "https://github.com/sponsors/nzakas"}},+ "node_modules/@humanwhocodes/module-importer": {+ "version": "1.0.1",+ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",+ "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",+ "dev": true,+ "engines": {+ "node": ">=12.22"+ },+ "funding": {+ "type": "github",+ "url": "https://github.com/sponsors/nzakas"+ }+ },"node_modules/@humanwhocodes/object-schema": {"version": "1.2.1","resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",@@ -5846,12 +5870,12 @@}},"node_modules/commitlint": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.0.3.tgz",- "integrity": "sha512-/KbIyrd6nmrRvu5zj8KKrjoC4z5V6hBmYphHgCFu75kPjHODg1XTtGFgbnb0AdSGBHlGMzmDvykO7ETs8wBKFg==",+ "version": "17.1.2",+ "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.1.2.tgz",+ "integrity": "sha512-ykPXC3TUfYlL8tqz/VOeaJpTPNrcdKrR4Y7ShxjF6l8SlBN/+4YhBJpomG2dx8Ac43FM9OE5rdn1+h7NxsIcAQ==","dev": true,"dependencies": {- "@commitlint/cli": "^17.0.3",+ "@commitlint/cli": "^17.1.2","@commitlint/types": "^17.0.0"},"bin": {@@ -6164,14 +6188,10 @@}},"node_modules/cosmiconfig-typescript-loader": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-2.0.2.tgz",- "integrity": "sha512-KmE+bMjWMXJbkWCeY4FJX/npHuZPNr9XF9q9CIQ/bpFwi1qHfCmSiKarrCcRa0LO4fWjk93pVoeRtJAkTGcYNw==",+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.0.0.tgz",+ "integrity": "sha512-cVpucSc2Tf+VPwCCR7SZzmQTQkPbkk4O01yXsYqXBIbjE1bhwqSyAgYQkRK1un4i0OPziTleqFhdkmOc4RQ/9g==","dev": true,- "dependencies": {- "cosmiconfig": "^7",- "ts-node": "^10.8.1"- },"engines": {"node": ">=12","npm": ">=6"@@ -6179,6 +6199,7 @@"peerDependencies": {"@types/node": "*","cosmiconfig": ">=7",+ "ts-node": ">=10","typescript": ">=3"}},@@ -7077,14 +7098,15 @@}},"node_modules/eslint": {- "version": "8.22.0",- "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz",- "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==",+ "version": "8.23.0",+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.23.0.tgz",+ "integrity": "sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA==","dev": true,"dependencies": {- "@eslint/eslintrc": "^1.3.0",+ "@eslint/eslintrc": "^1.3.1","@humanwhocodes/config-array": "^0.10.4","@humanwhocodes/gitignore-to-minimatch": "^1.0.2",+ "@humanwhocodes/module-importer": "^1.0.1","ajv": "^6.10.0","chalk": "^4.0.0","cross-spawn": "^7.0.2",@@ -7094,7 +7116,7 @@"eslint-scope": "^7.1.1","eslint-utils": "^3.0.0","eslint-visitor-keys": "^3.3.0",- "espree": "^9.3.3",+ "espree": "^9.4.0","esquery": "^1.4.0","esutils": "^2.0.2","fast-deep-equal": "^3.1.3",@@ -7119,8 +7141,7 @@"regexpp": "^3.2.0","strip-ansi": "^6.0.1","strip-json-comments": "^3.1.0",- "text-table": "^0.2.0",- "v8-compile-cache": "^2.0.3"+ "text-table": "^0.2.0"},"bin": {"eslint": "bin/eslint.js"@@ -7537,9 +7558,9 @@}},"node_modules/espree": {- "version": "9.3.3",- "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz",- "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==",+ "version": "9.4.0",+ "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz",+ "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==","dev": true,"dependencies": {"acorn": "^8.8.0",@@ -15567,9 +15588,9 @@}},"node_modules/typescript": {- "version": "4.7.4",- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",+ "version": "4.8.2",+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==","dev": true,"bin": {"tsc": "bin/tsc",@@ -16576,7 +16597,8 @@"got-scraping": "^3.2.9","iconv-lite": "^0.6.3","mime-types": "^2.1.35",- "ow": "^0.28.1"+ "ow": "^0.28.1",+ "type-fest": "^2.19.0"},"engines": {"node": ">=16.0.0"@@ -16593,6 +16615,17 @@"node": ">=0.10.0"}},+ "packages/http-crawler/node_modules/type-fest": {+ "version": "2.19.0",+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz",+ "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==",+ "engines": {+ "node": ">=12.20"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },"packages/jsdom-crawler": {"name": "@crawlee/jsdom","version": "3.0.4",@@ -17562,15 +17595,15 @@"dev": true},"@commitlint/cli": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.0.3.tgz",- "integrity": "sha512-oAo2vi5d8QZnAbtU5+0cR2j+A7PO8zuccux65R/EycwvsZrDVyW518FFrnJK2UQxbRtHFFIG+NjQ6vOiJV0Q8A==",+ "version": "17.1.2",+ "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.1.2.tgz",+ "integrity": "sha512-h/4Hlka3bvCLbnxf0Er2ri5A44VMlbMSkdTRp8Adv2tRiklSTRIoPGs7OEXDv3EoDs2AAzILiPookgM4Gi7LOw==","dev": true,"requires": {"@commitlint/format": "^17.0.0",- "@commitlint/lint": "^17.0.3",- "@commitlint/load": "^17.0.3",- "@commitlint/read": "^17.0.0",+ "@commitlint/lint": "^17.1.0",+ "@commitlint/load": "^17.1.2",+ "@commitlint/read": "^17.1.0","@commitlint/types": "^17.0.0","execa": "^5.0.0","lodash": "^4.17.19",@@ -17580,18 +17613,18 @@}},"@commitlint/config-conventional": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.0.3.tgz",- "integrity": "sha512-HCnzTm5ATwwwzNVq5Y57poS0a1oOOcd5pc1MmBpLbGmSysc4i7F/++JuwtdFPu16sgM3H9J/j2zznRLOSGVO2A==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.1.0.tgz",+ "integrity": "sha512-WU2p0c9/jLi8k2q2YrDV96Y8XVswQOceIQ/wyJvQxawJSCasLdRB3kUIYdNjOCJsxkpoUlV/b90ZPxp1MYZDiA==","dev": true,"requires": {"conventional-changelog-conventionalcommits": "^5.0.0"}},"@commitlint/config-validator": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.0.3.tgz",- "integrity": "sha512-3tLRPQJKapksGE7Kee9axv+9z5I2GDHitDH4q63q7NmNA0wkB+DAorJ0RHz2/K00Zb1/MVdHzhCga34FJvDihQ==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.1.0.tgz",+ "integrity": "sha512-Q1rRRSU09ngrTgeTXHq6ePJs2KrI+axPTgkNYDWSJIuS1Op4w3J30vUfSXjwn5YEJHklK3fSqWNHmBhmTR7Vdg==","dev": true,"requires": {"@commitlint/types": "^17.0.0",@@ -17625,9 +17658,9 @@}},"@commitlint/is-ignored": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.0.3.tgz",- "integrity": "sha512-/wgCXAvPtFTQZxsVxj7owLeRf5wwzcXLaYmrZPR4a87iD4sCvUIRl1/ogYrtOyUmHwWfQsvjqIB4mWE/SqWSnA==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.1.0.tgz",+ "integrity": "sha512-JITWKDMHhIh8IpdIbcbuH9rEQJty1ZWelgjleTFrVRAcEwN/sPzk1aVUXRIZNXMJWbZj8vtXRJnFihrml8uECQ==","dev": true,"requires": {"@commitlint/types": "^17.0.0",@@ -17635,34 +17668,43 @@}},"@commitlint/lint": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.0.3.tgz",- "integrity": "sha512-2o1fk7JUdxBUgszyt41sHC/8Nd5PXNpkmuOo9jvGIjDHzOwXyV0PSdbEVTH3xGz9NEmjohFHr5l+N+T9fcxong==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.1.0.tgz",+ "integrity": "sha512-ltpqM2ogt/+SDhUaScFo0MdscncEF96lvQTPMM/VTTWlw7sTGLLWkOOppsee2MN/uLNNWjQ7kqkd4h6JqoM9AQ==","dev": true,"requires": {- "@commitlint/is-ignored": "^17.0.3",+ "@commitlint/is-ignored": "^17.1.0","@commitlint/parse": "^17.0.0","@commitlint/rules": "^17.0.0","@commitlint/types": "^17.0.0"}},"@commitlint/load": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.0.3.tgz",- "integrity": "sha512-3Dhvr7GcKbKa/ey4QJ5MZH3+J7QFlARohUow6hftQyNjzoXXROm+RwpBes4dDFrXG1xDw9QPXA7uzrOShCd4bw==",+ "version": "17.1.2",+ "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.1.2.tgz",+ "integrity": "sha512-sk2p/jFYAWLChIfOIp/MGSIn/WzZ0vkc3afw+l4X8hGEYkvDe4gQUUAVxjl/6xMRn0HgnSLMZ04xXh5pkTsmgg==","dev": true,"requires": {- "@commitlint/config-validator": "^17.0.3",+ "@commitlint/config-validator": "^17.1.0","@commitlint/execute-rule": "^17.0.0",- "@commitlint/resolve-extends": "^17.0.3",+ "@commitlint/resolve-extends": "^17.1.0","@commitlint/types": "^17.0.0",- "@types/node": ">=12",+ "@types/node": "^14.0.0","chalk": "^4.1.0","cosmiconfig": "^7.0.0",- "cosmiconfig-typescript-loader": "^2.0.0",+ "cosmiconfig-typescript-loader": "^4.0.0","lodash": "^4.17.19","resolve-from": "^5.0.0",+ "ts-node": "^10.8.1","typescript": "^4.6.4"+ },+ "dependencies": {+ "@types/node": {+ "version": "14.18.26",+ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.26.tgz",+ "integrity": "sha512-0b+utRBSYj8L7XAp0d+DX7lI4cSmowNaaTkk6/1SKzbKkG+doLuPusB9EOvzLJ8ahJSk03bTLIL6cWaEd4dBKA==",+ "dev": true+ }}},"@commitlint/message": {@@ -17683,24 +17725,25 @@}},"@commitlint/read": {- "version": "17.0.0",- "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.0.0.tgz",- "integrity": "sha512-zkuOdZayKX3J6F6mPnVMzohK3OBrsEdOByIqp4zQjA9VLw1hMsDEFQ18rKgUc2adkZar+4S01QrFreDCfZgbxA==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.1.0.tgz",+ "integrity": "sha512-73BoFNBA/3Ozo2JQvGsE0J8SdrJAWGfZQRSHqvKaqgmY042Su4gXQLqvAzgr55S9DI1l9TiU/5WDuh8IE86d/g==","dev": true,"requires": {"@commitlint/top-level": "^17.0.0","@commitlint/types": "^17.0.0","fs-extra": "^10.0.0",- "git-raw-commits": "^2.0.0"+ "git-raw-commits": "^2.0.0",+ "minimist": "^1.2.6"}},"@commitlint/resolve-extends": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.0.3.tgz",- "integrity": "sha512-H/RFMvrcBeJCMdnVC4i8I94108UDccIHrTke2tyQEg9nXQnR5/Hd6MhyNWkREvcrxh9Y+33JLb+PiPiaBxCtBA==",+ "version": "17.1.0",+ "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.1.0.tgz",+ "integrity": "sha512-jqKm00LJ59T0O8O4bH4oMa4XyJVEOK4GzH8Qye9XKji+Q1FxhZznxMV/bDLyYkzbTodBt9sL0WLql8wMtRTbqQ==","dev": true,"requires": {- "@commitlint/config-validator": "^17.0.3",+ "@commitlint/config-validator": "^17.1.0","@commitlint/types": "^17.0.0","import-fresh": "^3.0.0","lodash": "^4.17.19",@@ -17861,7 +17904,8 @@"got-scraping": "^3.2.9","iconv-lite": "^0.6.3","mime-types": "^2.1.35",- "ow": "^0.28.1"+ "ow": "^0.28.1",+ "type-fest": "*"},"dependencies": {"iconv-lite": {@@ -17871,6 +17915,11 @@"requires": {"safer-buffer": ">= 2.1.2 < 3.0.0"}+ },+ "type-fest": {+ "version": "2.19.0",+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz",+ "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="}}},@@ -18144,14 +18193,14 @@}},"@eslint/eslintrc": {- "version": "1.3.0",- "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz",- "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==",+ "version": "1.3.1",+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.1.tgz",+ "integrity": "sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ==","dev": true,"requires": {"ajv": "^6.12.4","debug": "^4.3.2",- "espree": "^9.3.2",+ "espree": "^9.4.0","globals": "^13.15.0","ignore": "^5.2.0","import-fresh": "^3.2.1",@@ -18203,6 +18252,12 @@"integrity": "sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA==","dev": true},+ "@humanwhocodes/module-importer": {+ "version": "1.0.1",+ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",+ "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",+ "dev": true+ },"@humanwhocodes/object-schema": {"version": "1.2.1","resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",@@ -21933,12 +21988,12 @@"dev": true},"commitlint": {- "version": "17.0.3",- "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.0.3.tgz",- "integrity": "sha512-/KbIyrd6nmrRvu5zj8KKrjoC4z5V6hBmYphHgCFu75kPjHODg1XTtGFgbnb0AdSGBHlGMzmDvykO7ETs8wBKFg==",+ "version": "17.1.2",+ "resolved": "https://registry.npmjs.org/commitlint/-/commitlint-17.1.2.tgz",+ "integrity": "sha512-ykPXC3TUfYlL8tqz/VOeaJpTPNrcdKrR4Y7ShxjF6l8SlBN/+4YhBJpomG2dx8Ac43FM9OE5rdn1+h7NxsIcAQ==","dev": true,"requires": {- "@commitlint/cli": "^17.0.3",+ "@commitlint/cli": "^17.1.2","@commitlint/types": "^17.0.0"}},@@ -22190,14 +22245,11 @@}},"cosmiconfig-typescript-loader": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-2.0.2.tgz",- "integrity": "sha512-KmE+bMjWMXJbkWCeY4FJX/npHuZPNr9XF9q9CIQ/bpFwi1qHfCmSiKarrCcRa0LO4fWjk93pVoeRtJAkTGcYNw==",+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.0.0.tgz",+ "integrity": "sha512-cVpucSc2Tf+VPwCCR7SZzmQTQkPbkk4O01yXsYqXBIbjE1bhwqSyAgYQkRK1un4i0OPziTleqFhdkmOc4RQ/9g==","dev": true,- "requires": {- "cosmiconfig": "^7",- "ts-node": "^10.8.1"- }+ "requires": {}},"crawlee": {"version": "file:packages/crawlee",@@ -22890,14 +22942,15 @@}},"eslint": {- "version": "8.22.0",- "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.22.0.tgz",- "integrity": "sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA==",+ "version": "8.23.0",+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.23.0.tgz",+ "integrity": "sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA==","dev": true,"requires": {- "@eslint/eslintrc": "^1.3.0",+ "@eslint/eslintrc": "^1.3.1","@humanwhocodes/config-array": "^0.10.4","@humanwhocodes/gitignore-to-minimatch": "^1.0.2",+ "@humanwhocodes/module-importer": "^1.0.1","ajv": "^6.10.0","chalk": "^4.0.0","cross-spawn": "^7.0.2",@@ -22907,7 +22960,7 @@"eslint-scope": "^7.1.1","eslint-utils": "^3.0.0","eslint-visitor-keys": "^3.3.0",- "espree": "^9.3.3",+ "espree": "^9.4.0","esquery": "^1.4.0","esutils": "^2.0.2","fast-deep-equal": "^3.1.3",@@ -22932,8 +22985,7 @@"regexpp": "^3.2.0","strip-ansi": "^6.0.1","strip-json-comments": "^3.1.0",- "text-table": "^0.2.0",- "v8-compile-cache": "^2.0.3"+ "text-table": "^0.2.0"},"dependencies": {"ajv": {@@ -23246,9 +23298,9 @@"dev": true},"espree": {- "version": "9.3.3",- "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.3.tgz",- "integrity": "sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng==",+ "version": "9.4.0",+ "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz",+ "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==","dev": true,"requires": {"acorn": "^8.8.0",@@ -29237,9 +29289,9 @@}},"typescript": {- "version": "4.7.4",- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",+ "version": "4.8.2",+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==","dev": true},"uglify-js": {diff --git a/package.json b/package.jsonindex 25065af48..0f51fccd0 100644--- a/package.json+++ b/package.json@@ -52,9 +52,9 @@},"devDependencies": {"@apify/eslint-config-ts": "^0.2.3",- "@apify/log": "^2.0.0",+ "@apify/log": "^2.1.3","@apify/tsconfig": "^0.1.0",- "@commitlint/config-conventional": "^17.0.2",+ "@commitlint/config-conventional": "^17.1.0","@types/content-type": "^1.1.5","@types/deep-equal": "^1.0.1","@types/domhandler": "^2.4.2",@@ -62,14 +62,14 @@"@types/fs-extra": "^9.0.13","@types/htmlparser2": "^3.10.3","@types/inquirer": "^8.2.1",- "@types/jest": "^28.1.1",+ "@types/jest": "^28.1.8","@types/lodash.merge": "^4.6.7","@types/mime-types": "^2.1.1",- "@types/node": "^18.0.3",+ "@types/node": "^18.7.13","@types/proper-lockfile": "^4.1.2","@types/ps-tree": "^1.1.2","@types/rimraf": "^3.0.2",- "@types/semver": "^7.3.9",+ "@types/semver": "^7.3.12","@types/stream-json": "^1.7.2","@types/tough-cookie": "^4.0.2","@types/ws": "^7.4.7",@@ -78,27 +78,27 @@"apify": "3.0.2","basic-auth-parser": "^0.0.2","body-parser": "^1.20.0",- "commitlint": "^17.0.2",+ "commitlint": "^17.1.2","deep-equal": "^2.0.5",- "eslint": "^8.17.0",+ "eslint": "^8.23.0","express": "^4.18.1","fs-extra": "^10.1.0","gen-esm-wrapper": "^1.1.3",- "globby": "^13.1.1",+ "globby": "^13.1.2","husky": "^8.0.1",- "jest": "^28.1.1",- "lerna": "^5.1.1",- "lint-staged": "^13.0.1",- "node-gyp": "^9.0.0",+ "jest": "^28.1.3",+ "lerna": "^5.4.3",+ "lint-staged": "^13.0.3",+ "node-gyp": "^9.1.0","playwright": "1.25.1","portastic": "^1.0.1","proxy": "^1.0.2","puppeteer": "15.1.1",- "ts-jest": "^28.0.4",- "ts-node": "^10.8.1",+ "ts-jest": "^28.0.8",+ "ts-node": "^10.9.1","turbo": "1.4.3",- "typescript": "^4.7.4",- "ws": "^7.5.8"+ "typescript": "^4.8.2",+ "ws": "^7.5.9"},"packageManager": "[email protected]"}diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 3473d336f..7e6cc2fbb 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -55,7 +55,7 @@ export interface BrowserCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptiexport type BrowserHook<Context = BrowserCrawlingContext,- GoToOptions extends Record<PropertyKey, any> | undefined = Dictionary+ GoToOptions extends Dictionary | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;export interface BrowserCrawlerOptions<@@ -285,9 +285,9 @@ export interface BrowserCrawlerOptions<*/export abstract class BrowserCrawler<InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,- LaunchOptions = Dictionary,+ LaunchOptions extends Dictionary | undefined = Dictionary,Context extends BrowserCrawlingContext = BrowserCrawlingContext,- GoToOptions extends Record<PropertyKey, any> = Dictionary+ GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context> {/*** A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex 8cc2aed9e..016fc815e 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -69,7 +69,7 @@ export abstract class BrowserLauncher<Plugin extends BrowserPlugin,Launcher = Plugin['library'],T extends Constructor<Plugin> = Constructor<Plugin>,- LaunchOptions = Partial<Parameters<Plugin['launch']>[0]>,+ LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>,LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>,> {launcher: Launcher;diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.tsindex 2a52c29cd..42f8e4020 100644--- a/packages/browser-pool/src/abstract-classes/browser-controller.ts+++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts@@ -1,7 +1,7 @@import { nanoid } from 'nanoid';import { TypedEmitter } from 'tiny-typed-emitter';import { tryCancel } from '@apify/timeout';-import type { Cookie } from '@crawlee/types';+import type { Cookie, Dictionary } from '@crawlee/types';import { BROWSER_CONTROLLER_EVENTS } from '../events';import type { LaunchContext } from '../launch-context';import { log } from '../logger';@@ -13,7 +13,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;export interface BrowserControllerEvents<Library extends CommonLibrary,- LibraryOptions = Parameters<Library['launch']>[0],+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,@@ -32,7 +32,7 @@ export interface BrowserControllerEvents<*/export abstract class BrowserController<Library extends CommonLibrary = CommonLibrary,- LibraryOptions = Parameters<Library['launch']>[0],+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex 4eedb5c5a..e18fd6a2a 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -1,4 +1,5 @@import merge from 'lodash.merge';+import type { Dictionary } from '@crawlee/types';import type { LaunchContextOptions } from '../launch-context';import { LaunchContext } from '../launch-context';import type { BrowserController } from './browser-controller';@@ -24,7 +25,7 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/5*/export interface CommonLibrary {product?: string;- launch(...args: unknown[]): Promise<CommonBrowser>;+ launch(opts?: Dictionary): Promise<CommonBrowser>;name?: () => string;}@@ -74,7 +75,7 @@ export interface BrowserPluginOptions<LibraryOptions> {export interface CreateLaunchContextOptions<Library extends CommonLibrary,- LibraryOptions = Parameters<Library['launch']>[0],+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,@@ -88,7 +89,7 @@ export interface CreateLaunchContextOptions<*/export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary,- LibraryOptions = Parameters<Library['launch']>[0],+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,@@ -163,7 +164,9 @@ export abstract class BrowserPlugin<async launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = this.createLaunchContext(),): Promise<LaunchResult> {- const { proxyUrl, launchOptions }: { proxyUrl?: string; launchOptions: Record<string, any> } = launchContext;+ launchContext.launchOptions ??= {} as LibraryOptions;++ const { proxyUrl, launchOptions } = launchContext;if (proxyUrl) {await this._addProxyToLaunchOptions(launchContext);@@ -171,20 +174,20 @@ export abstract class BrowserPlugin<if (this._isChromiumBasedBrowser(launchContext)) {// This will set the args for chromium based browsers to hide the webdriver.- launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);+ (launchOptions as Dictionary).args = this._mergeArgsToHideWebdriver(launchOptions!.args);// When User-Agent is not set, and we're using Chromium in headless mode,// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,// as otherwise 'HeadlessChrome' is present in User-Agent string.- const userAgent = launchOptions.args.find((arg: string) => arg.startsWith('--user-agent'));- if (launchOptions.headless && !launchContext.fingerprint && !userAgent) {- launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);+ const userAgent = launchOptions!.args.find((arg: string) => arg.startsWith('--user-agent'));+ if (launchOptions!.headless && !launchContext.fingerprint && !userAgent) {+ launchOptions!.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);}}return this._launch(launchContext);}- private _mergeArgsToHideWebdriver(originalArgs: string[]): string[] {+ private _mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {if (!originalArgs?.length) {return ['--disable-blink-features=AutomationControlled'];}diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.tsindex 35dfd21ea..8988066a9 100644--- a/packages/browser-pool/src/launch-context.ts+++ b/packages/browser-pool/src/launch-context.ts@@ -1,3 +1,4 @@+import type { Dictionary } from '@crawlee/types';import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './abstract-classes/browser-plugin';import type { UnwrapPromise } from './utils';@@ -11,7 +12,7 @@ import type { UnwrapPromise } from './utils';*/export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibrary,- LibraryOptions = Parameters<Library['launch']>[0],+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,@@ -52,7 +53,7 @@ export interface LaunchContextOptions<export class LaunchContext<Library extends CommonLibrary = CommonLibrary,- LibraryOptions = Parameters<Library['launch']>[0],+ LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex a2fbe07d0..c9748ec5e 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -1,5 +1,4 @@import type { Dictionary } from '@crawlee/types';-// eslint isn't compatible with `import type`import type Puppeteer from './puppeteer-proxy-per-page';import type { Browser, Target, BrowserContext } from './puppeteer-proxy-per-page';import type { BrowserController } from '../abstract-classes/browser-controller';diff --git a/packages/cli/src/commands/CreateProjectCommand.ts b/packages/cli/src/commands/CreateProjectCommand.tsindex 0582278ec..f4c626616 100644--- a/packages/cli/src/commands/CreateProjectCommand.ts+++ b/packages/cli/src/commands/CreateProjectCommand.ts@@ -33,7 +33,7 @@ function rewrite(path: string, replacer: (from: string) => string): void {export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectArgs> {command = 'create [project-name]';describe = 'Creates a new Crawlee project directory from a selected boilerplate template.';- builder = async (args: Argv) => {+ builder = async (args: Argv<T>) => {const manifest = await fetchManifest();const choices = manifest.templates.map((t) => t.name);diff --git a/packages/cli/src/commands/RunProjectCommand.ts b/packages/cli/src/commands/RunProjectCommand.tsindex 9afc834ec..e69f66d94 100644--- a/packages/cli/src/commands/RunProjectCommand.ts+++ b/packages/cli/src/commands/RunProjectCommand.ts@@ -9,7 +9,7 @@ interface RunProjectArgs {export class RunProjectCommand<T> implements CommandModule<T, RunProjectArgs> {command = 'run';describe = 'Run crawlee project';- builder = async (args: Argv) => {+ builder = async (args: Argv<T>) => {args.option('purge', {alias: 't',default: true,diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 90b37de37..81153a149 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -104,7 +104,7 @@ export class KeyValueStore {private persistStateEventStarted = false;/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */- private readonly cache = new Map<string, unknown>();+ private readonly cache = new Map<string, Dictionary>();/*** @internal@@ -160,7 +160,7 @@ export class KeyValueStore {}const value = await this.getValue<T>(key, defaultValue);- this.cache.set(key, value);+ this.cache.set(key, value!);this.ensurePersistStateEvent();return value!;@@ -241,9 +241,9 @@ export class KeyValueStore {const optionsCopy = { ...options };// If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.- const cachedValue = this.cache.get(key) as T;+ const cachedValue = this.cache.get(key);- if (this.cache.has(key) && cachedValue !== value) {+ if (cachedValue && cachedValue !== value) {if (value === null) {// Cached state can be only object, so a propagation of `null` means removing all its properties.Object.keys(cachedValue).forEach((k) => this.cache.delete(k));diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 220bad699..5727f03c1 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -62,6 +62,7 @@"got-scraping": "^3.2.9","iconv-lite": "^0.6.3","mime-types": "^2.1.35",- "ow": "^0.28.1"+ "ow": "^0.28.1",+ "type-fest": "^2.19.0"}}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 28777bcbc..b85ca5e19 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -25,7 +25,8 @@ import contentTypeParser from 'content-type';import mime from 'mime-types';import type { OptionsInit, Method, Request as GotRequest, Response as GotResponse, GotOptionsInit } from 'got-scraping';import { gotScraping, TimeoutError } from 'got-scraping';-import { extname } from 'path';+import type { JsonValue } from 'type-fest';+import { extname } from 'node:path';import type { IncomingHttpHeaders, IncomingMessage } from 'node:http';import iconv from 'iconv-lite';import ow from 'ow';@@ -49,7 +50,7 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS: AutoscaledPoolOptions = {export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext> extends BasicCrawlerOptions<Context> {@@ -155,7 +156,7 @@ export type InternalHttpHook<Context> = (export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;/**@@ -163,7 +164,7 @@ export type HttpHook<*/export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawlerCrawler = HttpCrawler<any>> extends CrawlingContext<UserData> {/**@@ -188,12 +189,12 @@ export interface InternalHttpCrawlingContext<sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;}-export interface HttpCrawlingContext<UserData = any, JSONData = any>+export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any>extends InternalHttpCrawlingContext<UserData, JSONData, HttpCrawler<HttpCrawlingContext<UserData, JSONData>>> {}export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler- JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;/**diff --git a/website/package-lock.json b/website/package-lock.jsonindex 384d5d065..a1675d8f8 100644--- a/website/package-lock.json+++ b/website/package-lock.json@@ -32,7 +32,8 @@"got": "^11.8.5","path-browserify": "^1.0.1","prettier": "^2.6.2",- "rimraf": "^3.0.2"+ "rimraf": "^3.0.2",+ "typescript": "^4.8.2"}},"node_modules/@algolia/autocomplete-core": {@@ -12471,13 +12472,13 @@}},"node_modules/shiki": {- "version": "0.10.1",- "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.10.1.tgz",- "integrity": "sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==",+ "version": "0.11.1",+ "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.11.1.tgz",+ "integrity": "sha512-EugY9VASFuDqOexOgXR18ZV+TbFrQHeCpEYaXamO+SZlsnT/2LxuLBX25GGtIrwaEVFXUAbUQ601SWE2rMwWHA==","dependencies": {"jsonc-parser": "^3.0.0","vscode-oniguruma": "^1.6.1",- "vscode-textmate": "5.2.0"+ "vscode-textmate": "^6.0.0"}},"node_modules/side-channel": {@@ -13260,14 +13261,14 @@}},"node_modules/typedoc": {- "version": "0.23.10",- "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.10.tgz",- "integrity": "sha512-03EUiu/ZuScUBMnY6p0lY+HTH8SwhzvRE3gImoemdPDWXPXlks83UGTx++lyquWeB1MTwm9D9Ca8RIjkK3AFfQ==",+ "version": "0.23.11",+ "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.11.tgz",+ "integrity": "sha512-FhZ2HfqlS++53UwHk4txCsTrTlpYR0So/0osMyBeP1E7llRNRqycJGfYK1qx9Wvvv5VO8tGdpwzOwDW5FrTi7A==","dependencies": {"lunr": "^2.3.9",- "marked": "^4.0.18",+ "marked": "^4.0.19","minimatch": "^5.1.0",- "shiki": "^0.10.1"+ "shiki": "^0.11.1"},"bin": {"typedoc": "bin/typedoc"@@ -13276,7 +13277,7 @@"node": ">= 14.14"},"peerDependencies": {- "typescript": "4.6.x || 4.7.x"+ "typescript": "4.6.x || 4.7.x || 4.8.x"}},"node_modules/typedoc/node_modules/brace-expansion": {@@ -13299,10 +13300,9 @@}},"node_modules/typescript": {- "version": "4.7.4",- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",- "peer": true,+ "version": "4.8.2",+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==","bin": {"tsc": "bin/tsc","tsserver": "bin/tsserver"@@ -13943,9 +13943,9 @@"integrity": "sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA=="},"node_modules/vscode-textmate": {- "version": "5.2.0",- "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz",- "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ=="+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-6.0.0.tgz",+ "integrity": "sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ=="},"node_modules/wait-on": {"version": "6.0.1",@@ -23618,13 +23618,13 @@}},"shiki": {- "version": "0.10.1",- "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.10.1.tgz",- "integrity": "sha512-VsY7QJVzU51j5o1+DguUd+6vmCmZ5v/6gYu4vyYAhzjuNQU6P/vmSy4uQaOhvje031qQMiW0d2BwgMH52vqMng==",+ "version": "0.11.1",+ "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.11.1.tgz",+ "integrity": "sha512-EugY9VASFuDqOexOgXR18ZV+TbFrQHeCpEYaXamO+SZlsnT/2LxuLBX25GGtIrwaEVFXUAbUQ601SWE2rMwWHA==","requires": {"jsonc-parser": "^3.0.0","vscode-oniguruma": "^1.6.1",- "vscode-textmate": "5.2.0"+ "vscode-textmate": "^6.0.0"}},"side-channel": {@@ -24194,14 +24194,14 @@}},"typedoc": {- "version": "0.23.10",- "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.10.tgz",- "integrity": "sha512-03EUiu/ZuScUBMnY6p0lY+HTH8SwhzvRE3gImoemdPDWXPXlks83UGTx++lyquWeB1MTwm9D9Ca8RIjkK3AFfQ==",+ "version": "0.23.11",+ "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.23.11.tgz",+ "integrity": "sha512-FhZ2HfqlS++53UwHk4txCsTrTlpYR0So/0osMyBeP1E7llRNRqycJGfYK1qx9Wvvv5VO8tGdpwzOwDW5FrTi7A==","requires": {"lunr": "^2.3.9",- "marked": "^4.0.18",+ "marked": "^4.0.19","minimatch": "^5.1.0",- "shiki": "^0.10.1"+ "shiki": "^0.11.1"},"dependencies": {"brace-expansion": {@@ -24223,10 +24223,9 @@}},"typescript": {- "version": "4.7.4",- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",- "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",- "peer": true+ "version": "4.8.2",+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.2.tgz",+ "integrity": "sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw=="},"ua-parser-js": {"version": "0.7.31",@@ -24647,9 +24646,9 @@"integrity": "sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA=="},"vscode-textmate": {- "version": "5.2.0",- "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz",- "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ=="+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-6.0.0.tgz",+ "integrity": "sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ=="},"wait-on": {"version": "6.0.1",diff --git a/website/package.json b/website/package.jsonindex 1f26e8d02..ebd3afd32 100644--- a/website/package.json+++ b/website/package.json@@ -25,7 +25,8 @@"got": "^11.8.5","path-browserify": "^1.0.1","prettier": "^2.6.2",- "rimraf": "^3.0.2"+ "rimraf": "^3.0.2",+ "typescript": "^4.8.2"},"dependencies": {"@docusaurus/core": "2.0.1",