Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: wait before scrolling fragments into viewport (#2517)

Jindřich Bär0976e188ea24
diff --git a/website/package.json b/website/package.json
index 903c4b945..aad0d5ede 100644
--- a/website/package.json
+++ b/website/package.json
@@ -1,6 +1,7 @@
{
"scripts": {
"examples": "docusaurus-examples",
+ "postinstall": "npx patch-package",
"start": "rimraf .docusaurus && docusaurus start",
"start:fast": "rimraf .docusaurus && CRAWLEE_DOCS_FAST=1 docusaurus start",
"build": "rimraf .docusaurus && node --max_old_space_size=16000 node_modules/@docusaurus/core/bin/docusaurus.mjs build",
@@ -25,6 +26,7 @@
"eslint-plugin-react": "^7.32.2",
"eslint-plugin-react-hooks": "^4.6.0",
"fs-extra": "^11.1.0",
+ "patch-package": "^8.0.0",
"path-browserify": "^1.0.1",
"prettier": "^3.0.0",
"rimraf": "^5.0.0",
diff --git a/website/patches/@docusaurus+core+3.4.0.patch b/website/patches/@docusaurus+core+3.4.0.patch
new file mode 100644
index 000000000..220b45009
--- /dev/null
+++ b/website/patches/@docusaurus+core+3.4.0.patch
@@ -0,0 +1,19 @@
+diff --git a/node_modules/@docusaurus/core/lib/client/ClientLifecyclesDispatcher.js b/node_modules/@docusaurus/core/lib/client/ClientLifecyclesDispatcher.js
+index 903f8dc..b6b60bf 100644
+--- a/node_modules/@docusaurus/core/lib/client/ClientLifecyclesDispatcher.js
++++ b/node_modules/@docusaurus/core/lib/client/ClientLifecyclesDispatcher.js
+@@ -30,9 +30,11 @@ function scrollAfterNavigation({ location, previousLocation, }) {
+ window.scrollTo(0, 0);
+ }
+ else {
+- const id = decodeURIComponent(hash.substring(1));
+- const element = document.getElementById(id);
+- element?.scrollIntoView();
++ setTimeout(() => {
++ const id = decodeURIComponent(hash.substring(1));
++ const element = document.getElementById(id);
++ element?.scrollIntoView();
++ }, 100);
+ }
+ }
+ function ClientLifecyclesDispatcher({ children, location, previousLocation, }) {
diff --git a/website/yarn.lock b/website/yarn.lock
index a95d77974..63d51b34b 100644
--- a/website/yarn.lock
+++ b/website/yarn.lock
@@ -3646,6 +3646,13 @@ __metadata:
languageName: node
linkType: hard
+"@yarnpkg/lockfile@npm:^1.1.0":
+ version: 1.1.0
+ resolution: "@yarnpkg/lockfile@npm:1.1.0"
+ checksum: 10c0/0bfa50a3d756623d1f3409bc23f225a1d069424dbc77c6fd2f14fb377390cd57ec703dc70286e081c564be9051ead9ba85d81d66a3e68eeb6eb506d4e0c0fbda
+ languageName: node
+ linkType: hard
+
"abbrev@npm:^2.0.0":
version: 2.0.0
resolution: "abbrev@npm:2.0.0"
@@ -4726,7 +4733,7 @@ __metadata:
languageName: node
linkType: hard
-"ci-info@npm:^3.2.0":
+"ci-info@npm:^3.2.0, ci-info@npm:^3.7.0":
version: 3.9.0
resolution: "ci-info@npm:3.9.0"
checksum: 10c0/6f0109e36e111684291d46123d491bc4e7b7a1934c3a20dea28cba89f1d4a03acd892f5f6a81ed3855c38647e285a150e3c9ba062e38943bef57fee6c1554c3a
@@ -6855,6 +6862,15 @@ __metadata:
languageName: node
linkType: hard
+"find-yarn-workspace-root@npm:^2.0.0":
+ version: 2.0.0
+ resolution: "find-yarn-workspace-root@npm:2.0.0"
+ dependencies:
+ micromatch: "npm:^4.0.2"
+ checksum: 10c0/b0d3843013fbdaf4e57140e0165889d09fa61745c9e85da2af86e54974f4cc9f1967e40f0d8fc36a79d53091f0829c651d06607d552582e53976f3cd8f4e5689
+ languageName: node
+ linkType: hard
+
"flat-cache@npm:^3.0.4":
version: 3.2.0
resolution: "flat-cache@npm:3.2.0"
@@ -7329,7 +7345,7 @@ __metadata:
languageName: node
linkType: hard
-"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.11, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9":
+"graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.11, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9":
version: 4.2.11
resolution: "graceful-fs@npm:4.2.11"
checksum: 10c0/386d011a553e02bc594ac2ca0bd6d9e4c22d7fa8cfbfc448a6d148c59ea881b092db9dbe3547ae4b88e55f1b01f7c4a2ecc53b310c042793e63aa44cf6c257f2
@@ -8483,7 +8499,7 @@ __metadata:
languageName: node
linkType: hard
-"is-wsl@npm:^2.2.0":
+"is-wsl@npm:^2.1.1, is-wsl@npm:^2.2.0":
version: 2.2.0
resolution: "is-wsl@npm:2.2.0"
dependencies:
@@ -8716,6 +8732,18 @@ __metadata:
languageName: node
linkType: hard
+"json-stable-stringify@npm:^1.0.2":
+ version: 1.1.1
+ resolution: "json-stable-stringify@npm:1.1.1"
+ dependencies:
+ call-bind: "npm:^1.0.5"
+ isarray: "npm:^2.0.5"
+ jsonify: "npm:^0.0.1"
+ object-keys: "npm:^1.1.1"
+ checksum: 10c0/3801e3eeccbd030afb970f54bea690a079cfea7d9ed206a1b17ca9367f4b7772c764bf77a48f03e56b50e5f7ee7d11c52339fe20d8d7ccead003e4ca69e4cfde
+ languageName: node
+ linkType: hard
+
"json5@npm:^1.0.2":
version: 1.0.2
resolution: "json5@npm:1.0.2"
@@ -8756,6 +8784,13 @@ __metadata:
languageName: node
linkType: hard
+"jsonify@npm:^0.0.1":
+ version: 0.0.1
+ resolution: "jsonify@npm:0.0.1"
+ checksum: 10c0/7f5499cdd59a0967ed35bda48b7cec43d850bbc8fb955cdd3a1717bb0efadbe300724d5646de765bb7a99fc1c3ab06eb80d93503c6faaf99b4ff50a3326692f6
+ languageName: node
+ linkType: hard
+
"jsx-ast-utils@npm:^2.4.1 || ^3.0.0, jsx-ast-utils@npm:^3.3.5":
version: 3.3.5
resolution: "jsx-ast-utils@npm:3.3.5"
@@ -8784,6 +8819,15 @@ __metadata:
languageName: node
linkType: hard
+"klaw-sync@npm:^6.0.0":
+ version: 6.0.0
+ resolution: "klaw-sync@npm:6.0.0"
+ dependencies:
+ graceful-fs: "npm:^4.1.11"
+ checksum: 10c0/00d8e4c48d0d699b743b3b028e807295ea0b225caf6179f51029e19783a93ad8bb9bccde617d169659fbe99559d73fb35f796214de031d0023c26b906cecd70a
+ languageName: node
+ linkType: hard
+
"kleur@npm:^3.0.3":
version: 3.0.3
resolution: "kleur@npm:3.0.3"
@@ -10457,6 +10501,16 @@ __metadata:
languageName: node
linkType: hard
+"open@npm:^7.4.2":
+ version: 7.4.2
+ resolution: "open@npm:7.4.2"
+ dependencies:
+ is-docker: "npm:^2.0.0"
+ is-wsl: "npm:^2.1.1"
+ checksum: 10c0/77573a6a68f7364f3a19a4c80492712720746b63680ee304555112605ead196afe91052bd3c3d165efdf4e9d04d255e87de0d0a77acec11ef47fd5261251813f
+ languageName: node
+ linkType: hard
+
"open@npm:^8.0.9, open@npm:^8.4.0":
version: 8.4.2
resolution: "open@npm:8.4.2"
@@ -10491,6 +10545,13 @@ __metadata:
languageName: node
linkType: hard
+"os-tmpdir@npm:~1.0.2":
+ version: 1.0.2
+ resolution: "os-tmpdir@npm:1.0.2"
+ checksum: 10c0/f438450224f8e2687605a8dd318f0db694b6293c5d835ae509a69e97c8de38b6994645337e5577f5001115470414638978cc49da1cdcc25106dad8738dc69990
+ languageName: node
+ linkType: hard
+
"p-cancelable@npm:^3.0.0":
version: 3.0.0
resolution: "p-cancelable@npm:3.0.0"
@@ -10694,6 +10755,31 @@ __metadata:
languageName: node
linkType: hard
+"patch-package@npm:^8.0.0":
+ version: 8.0.0
+ resolution: "patch-package@npm:8.0.0"
+ dependencies:
+ "@yarnpkg/lockfile": "npm:^1.1.0"
+ chalk: "npm:^4.1.2"
+ ci-info: "npm:^3.7.0"
+ cross-spawn: "npm:^7.0.3"
+ find-yarn-workspace-root: "npm:^2.0.0"
+ fs-extra: "npm:^9.0.0"
+ json-stable-stringify: "npm:^1.0.2"
+ klaw-sync: "npm:^6.0.0"
+ minimist: "npm:^1.2.6"
+ open: "npm:^7.4.2"
+ rimraf: "npm:^2.6.3"
+ semver: "npm:^7.5.3"
+ slash: "npm:^2.0.0"
+ tmp: "npm:^0.0.33"
+ yaml: "npm:^2.2.2"
+ bin:
+ patch-package: index.js
+ checksum: 10c0/690eab0537e953a3fd7d32bb23f0e82f97cd448f8244c3227ed55933611a126f9476397325c06ad2c11d881a19b427a02bd1881bee78d89f1731373fc4fe0fee
+ languageName: node
+ linkType: hard
+
"path-browserify@npm:^1.0.1":
version: 1.0.1
resolution: "path-browserify@npm:1.0.1"
@@ -12193,6 +12279,17 @@ __metadata:
languageName: node
linkType: hard
+"rimraf@npm:^2.6.3":
+ version: 2.7.1
+ resolution: "rimraf@npm:2.7.1"
+ dependencies:
+ glob: "npm:^7.1.3"
+ bin:
+ rimraf: ./bin.js
+ checksum: 10c0/4eef73d406c6940927479a3a9dee551e14a54faf54b31ef861250ac815172bade86cc6f7d64a4dc5e98b65e4b18a2e1c9ff3b68d296be0c748413f092bb0dd40
+ languageName: node
+ linkType: hard
+
"rimraf@npm:^3.0.2":
version: 3.0.2
resolution: "rimraf@npm:3.0.2"
@@ -12253,6 +12350,7 @@ __metadata:
eslint-plugin-react: "npm:^7.32.2"
eslint-plugin-react-hooks: "npm:^4.6.0"
fs-extra: "npm:^11.1.0"
+ patch-package: "npm:^8.0.0"
path-browserify: "npm:^1.0.1"
prettier: "npm:^3.0.0"
prism-react-renderer: "npm:^2.1.0"
@@ -12438,7 +12536,7 @@ __metadata:
languageName: node
linkType: hard
-"semver@npm:^7.3.2, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.4, semver@npm:^7.6.0":
+"semver@npm:^7.3.2, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0":
version: 7.6.2
resolution: "semver@npm:7.6.2"
bin:
@@ -12703,6 +12801,13 @@ __metadata:
languageName: node
linkType: hard
+"slash@npm:^2.0.0":
+ version: 2.0.0
+ resolution: "slash@npm:2.0.0"
+ checksum: 10c0/f83dbd3cb62c41bb8fcbbc6bf5473f3234b97fa1d008f571710a9d3757a28c7169e1811cad1554ccb1cc531460b3d221c9a7b37f549398d9a30707f0a5af9193
+ languageName: node
+ linkType: hard
+
"slash@npm:^3.0.0":
version: 3.0.0
resolution: "slash@npm:3.0.0"
@@ -13257,6 +13362,15 @@ __metadata:
languageName: node
linkType: hard
+"tmp@npm:^0.0.33":
+ version: 0.0.33
+ resolution: "tmp@npm:0.0.33"
+ dependencies:
+ os-tmpdir: "npm:~1.0.2"
+ checksum: 10c0/69863947b8c29cabad43fe0ce65cec5bb4b481d15d4b4b21e036b060b3edbf3bc7a5541de1bacb437bb3f7c4538f669752627fdf9b4aaf034cebd172ba373408
+ languageName: node
+ linkType: hard
+
"to-fast-properties@npm:^2.0.0":
version: 2.0.0
resolution: "to-fast-properties@npm:2.0.0"
@@ -14225,6 +14339,15 @@ __metadata:
languageName: node
linkType: hard
+"yaml@npm:^2.2.2":
+ version: 2.4.3
+ resolution: "yaml@npm:2.4.3"
+ bin:
+ yaml: bin.mjs
+ checksum: 10c0/b4a9dea34265f000402c909144ac310be42c4526dfd16dff1aee2b04a0d94051713651c0cd2b0a3d8109266997422120f16a7934629d12f22dc215839ebbeccf
+ languageName: node
+ linkType: hard
+
"yocto-queue@npm:^0.1.0":
version: 0.1.0
resolution: "yocto-queue@npm:0.1.0"