Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add fingerprinting support (#1243)

petrpatekbd596def4140
diff --git a/package.json b/package.json
index 38c40b71d..43ecb43d0 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
"@apify/storage-local": "^2.0.1",
"@apify/utilities": "^1.1.2",
"apify-client": "^2.0.2",
- "browser-pool": "^3.0.4",
+ "browser-pool": "^3.1.0-beta.0",
"cheerio": "1.0.0-rc.10",
"content-type": "^1.0.4",
"got-scraping": "^3.2.5",
diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.js
index 0aa1f2450..9368bce20 100644
--- a/test/crawlers/browser_crawler.test.js
+++ b/test/crawlers/browser_crawler.test.js
@@ -570,6 +570,38 @@ describe('BrowserCrawler', () => {
await browserCrawler.browserPool.destroy();
});
+ test('should allow using fingerprints from browser pool', async () => {
+ const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://example.com/?q=1' },
+ ],
+ });
+ const browserCrawler = new Apify.BrowserCrawler({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ useFingerprints: true,
+ fingerprintsOptions: {
+ fingerprintGeneratorOptions: {
+ operatingSystems: ['windows'],
+ },
+ },
+ },
+ requestList,
+ useSessionPool: false,
+ gotoFunction: async ({ page, request }) => {
+ return page.goto(request.url);
+ },
+ handlePageFunction: async ({ browserController }) => {
+ expect(browserController.launchContext.fingerprint).toBeDefined();
+ },
+
+ });
+ await requestList.initialize();
+
+ await browserCrawler.run();
+ expect.hasAssertions();
+ });
+
describe('proxy', () => {
let requestList;
beforeEach(async () => {
diff --git a/tsconfig.json b/tsconfig.json
index c09189e59..1e8e73cb1 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -10,7 +10,8 @@
"allowJs": true,
"resolveJsonModule": false,
"noUnusedLocals": false,
- "outDir": "build"
+ "outDir": "build",
+ "skipLibCheck": true
},
"include": [
"./src/**/*"