Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix proxy rotation test (use mocked proxy servers)
Martin Adámekf59d95bffeee
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 2371ea1f0..642207bbe 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -361,20 +361,26 @@ describe('PuppeteerCrawler', () => {});test('proxy rotation', async () => {- const used = new Map();+ const proxies = new Set();+ const sessions = new Set();+ const serverUrl = `http://127.0.0.1:${target.address().port}`;+ const proxyConfiguration = new ProxyConfiguration({+ proxyUrls: [+ `http://127.0.0.2:${servers[0].port}`,+ `http://127.0.0.3:${servers[1].port}`,+ `http://127.0.0.4:${servers[2].port}`,+ ],+ });const puppeteerCrawler = new Apify.PuppeteerCrawler({requestList: await Apify.openRequestList(null, [- { url: 'https://api.apify.com/v2/browser-info', uniqueKey: '1' },- { url: 'https://api.apify.com/v2/browser-info', uniqueKey: '2' },- { url: 'https://api.apify.com/v2/browser-info', uniqueKey: '3' },- { url: 'https://api.apify.com/v2/browser-info', uniqueKey: '4' },- { url: 'https://api.apify.com/v2/browser-info', uniqueKey: '5' },+ { url: `${serverUrl}/?q=1` },+ { url: `${serverUrl}/?q=2` },+ { url: `${serverUrl}/?q=3` },]),launchContext: {launchOptions: {headless: true,},- // useIncognitoPages: true,},maxConcurrency: 1,sessionPoolOptions: {@@ -382,26 +388,16 @@ describe('PuppeteerCrawler', () => {maxUsageCount: 1,},},- proxyConfiguration: await Apify.createProxyConfiguration(),- async handlePageFunction(context) {- const { request, page } = context;-- // eslint-disable-next-line no-undef- const pageContent = await page.$eval('body', (el) => JSON.parse(el.textContent));- const { clientIp } = pageContent;-- if (used.has(clientIp)) throw new Error(`The ip address ${clientIp} was already used. Proxy rotation does not work properly.`)- used.set(clientIp, request.url);-- return {- url: request.url,- clientIp,- };+ proxyConfiguration,+ async handlePageFunction({ proxyInfo, session }) {+ proxies.add(proxyInfo.url);+ sessions.add(session.id);},});await puppeteerCrawler.run();- expect(used.size).toBe(5);+ expect(proxies.size).toBe(3); // 3 different proxies used+ expect(sessions.size).toBe(3); // 3 different sessions used});test('proxy per page', async () => {