Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add fingerprinting support (#1243)
petrpatekbd596def4140
diff --git a/package.json b/package.jsonindex 38c40b71d..43ecb43d0 100644--- a/package.json+++ b/package.json@@ -56,7 +56,7 @@"@apify/storage-local": "^2.0.1","@apify/utilities": "^1.1.2","apify-client": "^2.0.2",- "browser-pool": "^3.0.4",+ "browser-pool": "^3.1.0-beta.0","cheerio": "1.0.0-rc.10","content-type": "^1.0.4","got-scraping": "^3.2.5",diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.jsindex 0aa1f2450..9368bce20 100644--- a/test/crawlers/browser_crawler.test.js+++ b/test/crawlers/browser_crawler.test.js@@ -570,6 +570,38 @@ describe('BrowserCrawler', () => {await browserCrawler.browserPool.destroy();});+ test('should allow using fingerprints from browser pool', async () => {+ const requestList = new Apify.RequestList({+ sources: [+ { url: 'http://example.com/?q=1' },+ ],+ });+ const browserCrawler = new Apify.BrowserCrawler({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ useFingerprints: true,+ fingerprintsOptions: {+ fingerprintGeneratorOptions: {+ operatingSystems: ['windows'],+ },+ },+ },+ requestList,+ useSessionPool: false,+ gotoFunction: async ({ page, request }) => {+ return page.goto(request.url);+ },+ handlePageFunction: async ({ browserController }) => {+ expect(browserController.launchContext.fingerprint).toBeDefined();+ },++ });+ await requestList.initialize();++ await browserCrawler.run();+ expect.hasAssertions();+ });+describe('proxy', () => {let requestList;beforeEach(async () => {diff --git a/tsconfig.json b/tsconfig.jsonindex c09189e59..1e8e73cb1 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -10,7 +10,8 @@"allowJs": true,"resolveJsonModule": false,"noUnusedLocals": false,- "outDir": "build"+ "outDir": "build",+ "skipLibCheck": true},"include": ["./src/**/*"