Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: Check if the proxy tier drops after an amount of successful requests (#2490)
Jan Buchara2f39d9ecd20
diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.tsindex fedeb0667..205ca386a 100644--- a/test/core/proxy_configuration.test.ts+++ b/test/core/proxy_configuration.test.ts@@ -248,12 +248,11 @@ describe('ProxyConfiguration', () => {gotToTheHighestProxy = true;break;}- request.sessionRotationCount++;}expect(gotToTheHighestProxy).toBe(true);- // now let's go back down+ // Even the highest-tier proxies didn't help - we should try going downlet gotToTheLowestProxy = false;for (let i = 0; i < 20; i++) {@@ -262,7 +261,45 @@ describe('ProxyConfiguration', () => {gotToTheLowestProxy = true;break;}- // We don't increment the sessionRotationCount here - this causes the proxy tier to go down (current proxy is ok, so it tries to downshift in some time)+ }++ expect(gotToTheLowestProxy).toBe(true);+ });++ test('successful requests make the proxy tier drop eventually', async () => {+ const tieredProxyUrls = [['http://proxy.com:1111'], ['http://proxy.com:2222'], ['http://proxy.com:3333']];++ const proxyConfiguration = new ProxyConfiguration({+ tieredProxyUrls,+ });++ const failingRequest = new Request({+ url: 'http://example.com',+ });+ let gotToTheHighestProxy = false;++ for (let i = 0; i < 10; i++) {+ const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request: failingRequest });++ if (lastProxyUrl === tieredProxyUrls[2][0]) {+ gotToTheHighestProxy = true;+ break;+ }+ }++ expect(gotToTheHighestProxy).toBe(true);++ let gotToTheLowestProxy = false;++ for (let i = 0; i < 100; i++) {+ const lastProxyUrl = await proxyConfiguration.newUrl('session-id', {+ request: new Request({ url: `http://example.com/${i}` }),+ });++ if (lastProxyUrl === tieredProxyUrls[0][0]) {+ gotToTheLowestProxy = true;+ break;+ }}expect(gotToTheLowestProxy).toBe(true);