Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: update fingerprint suite
Martin Adámek42e1419879ed
diff --git a/package-lock.json b/package-lock.jsonindex bcc99f79e..54d65f1f1 100644--- a/package-lock.json+++ b/package-lock.json@@ -1030,14 +1030,14 @@"resolved": "packages/core","link": true},- "node_modules/@crawlee/jsdom": {- "resolved": "packages/jsdom-crawler",- "link": true- },"node_modules/@crawlee/http": {"resolved": "packages/http-crawler","link": true},+ "node_modules/@crawlee/jsdom": {+ "resolved": "packages/jsdom-crawler",+ "link": true+ },"node_modules/@crawlee/memory-storage": {"resolved": "packages/memory-storage","link": true@@ -7927,24 +7927,26 @@}},"node_modules/fingerprint-generator": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.0.2.tgz",- "integrity": "sha512-8hI0EHJXvaBFbGIGOummSBYEfQNYqUq2cfpKNucU/tk80Nn3k/Kxi/fgT85uWYx0CohtY0lI3pcyQZ8+6KmafA==",+ "version": "2.0.5",+ "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.0.5.tgz",+ "integrity": "sha512-cHQp+0gmqvX3mNv6Ggk3JJtut7bPZ00b3yAQhZkRmmctnEi5vrbx3mqESlbmBefKcaFYmFuWLt2xA3FFWYj+jg==","dependencies": {- "generative-bayesian-network": "^2.0.0",- "header-generator": "^2.0.0"+ "generative-bayesian-network": "^2.0.4",+ "header-generator": "^2.0.6",+ "tslib": "^2.4.0"},"engines": {"node": ">=16.0.0"}},"node_modules/fingerprint-injector": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.0.2.tgz",- "integrity": "sha512-H51T0eqoqKTw6ov+Bv1PTOliKXQpHvsb/6lf14w8hPvIGLhVVGHnxXHZ90z3NFGlfP998fFDVvgzH8nYlbmvXQ==",+ "version": "2.0.4",+ "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.0.4.tgz",+ "integrity": "sha512-bYvdmN/d9ugFIedoMn4T5Fssu6C0gzn4UQteBuFKhGsrV7gi/L+dpG6RZIgzn52neoRtbCh1SXXZZ247j+iRjw==","dependencies": {"@apify/log": "^1.0.5",- "fingerprint-generator": "^2.0.0"+ "fingerprint-generator": "^2.0.5",+ "tslib": "^2.4.0"},"engines": {"node": ">=16.0.0"@@ -8196,11 +8198,12 @@}},"node_modules/generative-bayesian-network": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.0.2.tgz",- "integrity": "sha512-954uYX7seQU5KcQgaA7g2yDJTq8WKGVj+gTDjKDJSUJ9Vd0bippgCmOp+SVwgNn+Zj+bb0BUc2qBud9yF+hYPg==",+ "version": "2.0.4",+ "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.0.4.tgz",+ "integrity": "sha512-oQG5ddL3qmTkkV7U6rnXJEiALXIpu4nNtByfvMM/VsaFF//yd5uesQauRt3KGIziUdtPxriDvof3q60LECnAPQ==","dependencies": {- "adm-zip": "^0.5.9"+ "adm-zip": "^0.5.9",+ "tslib": "^2.4.0"}},"node_modules/gensync": {@@ -8785,13 +8788,14 @@"dev": true},"node_modules/header-generator": {- "version": "2.0.3",- "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.0.3.tgz",- "integrity": "sha512-v3oKkmQtRSK9hawVJvMD0T5KAC0ggKnmcoSvBMDq0XXlSIPjjkAsMvde7LQu26bFjs+apvAmWbhPtum0LWkJlw==",+ "version": "2.0.6",+ "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.0.6.tgz",+ "integrity": "sha512-+5yyn4LQPV+WVx1tmAlVBOdKTp1nXKlz+MW5MWN3tdoSERS/QCQbYUm+jAgE98ZBuP8tcOgVkU2aLIpnfXuBJA==","dependencies": {"browserslist": "^4.21.1",- "generative-bayesian-network": "^2.0.0",- "ow": "^0.28.1"+ "generative-bayesian-network": "^2.0.4",+ "ow": "^0.28.1",+ "tslib": "^2.4.0"},"engines": {"node": ">=16.0.0"@@ -16358,8 +16362,8 @@"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0","@crawlee/types": "^3.0.3",- "fingerprint-generator": "^2.0.2-dev.3",- "fingerprint-injector": "^2.0.2-dev.3",+ "fingerprint-generator": "^2.0.5",+ "fingerprint-injector": "^2.0.4","lodash.merge": "^4.6.2","nanoid": "^3.3.4","ow": "^0.28.1",@@ -16485,8 +16489,8 @@"@crawlee/cheerio": "^3.0.3","@crawlee/cli": "^3.0.3","@crawlee/core": "^3.0.3",- "@crawlee/jsdom": "^3.0.3","@crawlee/http": "^3.0.3",+ "@crawlee/jsdom": "^3.0.3","@crawlee/playwright": "^3.0.3","@crawlee/puppeteer": "^3.0.3","@crawlee/utils": "^3.0.3",@@ -16511,21 +16515,6 @@}}},- "packages/jsdom-crawler": {- "name": "@crawlee/jsdom",- "version": "3.0.3",- "license": "Apache-2.0",- "dependencies": {- "@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.0.3",- "@crawlee/types": "^3.0.3",- "@types/jsdom": "^20.0.0",- "jsdom": "^20.0.0"- },- "engines": {- "node": ">=16.0.0"- }- },"packages/http-crawler": {"name": "@crawlee/http","version": "3.0.3",@@ -16557,6 +16546,21 @@"node": ">=0.10.0"}},+ "packages/jsdom-crawler": {+ "name": "@crawlee/jsdom",+ "version": "3.0.3",+ "license": "Apache-2.0",+ "dependencies": {+ "@apify/utilities": "^2.1.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@types/jsdom": "^20.0.0",+ "jsdom": "^20.0.0"+ },+ "engines": {+ "node": ">=16.0.0"+ }+ },"packages/memory-storage": {"name": "@crawlee/memory-storage","version": "3.0.3",@@ -17718,8 +17722,8 @@"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0","@crawlee/types": "^3.0.3",- "fingerprint-generator": "^2.0.2-dev.3",- "fingerprint-injector": "^2.0.2-dev.3",+ "fingerprint-generator": "^2.0.5",+ "fingerprint-injector": "^2.0.4","lodash.merge": "^4.6.2","nanoid": "^3.3.4","ow": "^0.28.1",@@ -17798,16 +17802,6 @@}}},- "@crawlee/jsdom": {- "version": "file:packages/jsdom-crawler",- "requires": {- "@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.0.3",- "@crawlee/types": "^3.0.3",- "@types/jsdom": "^20.0.0",- "jsdom": "^20.0.0"- }- },"@crawlee/http": {"version": "file:packages/http-crawler","requires": {@@ -17833,6 +17827,16 @@}}},+ "@crawlee/jsdom": {+ "version": "file:packages/jsdom-crawler",+ "requires": {+ "@apify/utilities": "^2.1.4",+ "@crawlee/http": "^3.0.3",+ "@crawlee/types": "^3.0.3",+ "@types/jsdom": "^20.0.0",+ "jsdom": "^20.0.0"+ }+ },"@crawlee/memory-storage": {"version": "file:packages/memory-storage","requires": {@@ -22124,8 +22128,8 @@"@crawlee/cheerio": "^3.0.3","@crawlee/cli": "^3.0.3","@crawlee/core": "^3.0.3",- "@crawlee/jsdom": "^3.0.3","@crawlee/http": "^3.0.3",+ "@crawlee/jsdom": "^3.0.3","@crawlee/playwright": "^3.0.3","@crawlee/puppeteer": "^3.0.3","@crawlee/utils": "^3.0.3",@@ -23539,21 +23543,23 @@}},"fingerprint-generator": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.0.2.tgz",- "integrity": "sha512-8hI0EHJXvaBFbGIGOummSBYEfQNYqUq2cfpKNucU/tk80Nn3k/Kxi/fgT85uWYx0CohtY0lI3pcyQZ8+6KmafA==",+ "version": "2.0.5",+ "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.0.5.tgz",+ "integrity": "sha512-cHQp+0gmqvX3mNv6Ggk3JJtut7bPZ00b3yAQhZkRmmctnEi5vrbx3mqESlbmBefKcaFYmFuWLt2xA3FFWYj+jg==","requires": {- "generative-bayesian-network": "^2.0.0",- "header-generator": "^2.0.0"+ "generative-bayesian-network": "^2.0.4",+ "header-generator": "^2.0.6",+ "tslib": "^2.4.0"}},"fingerprint-injector": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.0.2.tgz",- "integrity": "sha512-H51T0eqoqKTw6ov+Bv1PTOliKXQpHvsb/6lf14w8hPvIGLhVVGHnxXHZ90z3NFGlfP998fFDVvgzH8nYlbmvXQ==",+ "version": "2.0.4",+ "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.0.4.tgz",+ "integrity": "sha512-bYvdmN/d9ugFIedoMn4T5Fssu6C0gzn4UQteBuFKhGsrV7gi/L+dpG6RZIgzn52neoRtbCh1SXXZZ247j+iRjw==","requires": {"@apify/log": "^1.0.5",- "fingerprint-generator": "^2.0.0"+ "fingerprint-generator": "^2.0.5",+ "tslib": "^2.4.0"},"dependencies": {"@apify/consts": {@@ -23735,11 +23741,12 @@}},"generative-bayesian-network": {- "version": "2.0.2",- "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.0.2.tgz",- "integrity": "sha512-954uYX7seQU5KcQgaA7g2yDJTq8WKGVj+gTDjKDJSUJ9Vd0bippgCmOp+SVwgNn+Zj+bb0BUc2qBud9yF+hYPg==",+ "version": "2.0.4",+ "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.0.4.tgz",+ "integrity": "sha512-oQG5ddL3qmTkkV7U6rnXJEiALXIpu4nNtByfvMM/VsaFF//yd5uesQauRt3KGIziUdtPxriDvof3q60LECnAPQ==","requires": {- "adm-zip": "^0.5.9"+ "adm-zip": "^0.5.9",+ "tslib": "^2.4.0"}},"gensync": {@@ -24173,13 +24180,14 @@"dev": true},"header-generator": {- "version": "2.0.3",- "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.0.3.tgz",- "integrity": "sha512-v3oKkmQtRSK9hawVJvMD0T5KAC0ggKnmcoSvBMDq0XXlSIPjjkAsMvde7LQu26bFjs+apvAmWbhPtum0LWkJlw==",+ "version": "2.0.6",+ "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.0.6.tgz",+ "integrity": "sha512-+5yyn4LQPV+WVx1tmAlVBOdKTp1nXKlz+MW5MWN3tdoSERS/QCQbYUm+jAgE98ZBuP8tcOgVkU2aLIpnfXuBJA==","requires": {"browserslist": "^4.21.1",- "generative-bayesian-network": "^2.0.0",- "ow": "^0.28.1"+ "generative-bayesian-network": "^2.0.4",+ "ow": "^0.28.1",+ "tslib": "^2.4.0"}},"hosted-git-info": {diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 7aa136515..1fc4d21e9 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -39,8 +39,8 @@"@apify/log": "^2.0.0","@apify/timeout": "^0.3.0","@crawlee/types": "^3.0.3",- "fingerprint-generator": "^2.0.2-dev.3",- "fingerprint-injector": "^2.0.2-dev.3",+ "fingerprint-generator": "^2.0.5",+ "fingerprint-injector": "^2.0.4","lodash.merge": "^4.6.2","nanoid": "^3.3.4","ow": "^0.28.1",