Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: Check if the proxy tier drops after an amount of successful requests (#2490)

Jan Buchara2f39d9ecd20
diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.ts
index fedeb0667..205ca386a 100644
--- a/test/core/proxy_configuration.test.ts
+++ b/test/core/proxy_configuration.test.ts
@@ -248,12 +248,11 @@ describe('ProxyConfiguration', () => {
gotToTheHighestProxy = true;
break;
}
- request.sessionRotationCount++;
}
expect(gotToTheHighestProxy).toBe(true);
- // now let's go back down
+ // Even the highest-tier proxies didn't help - we should try going down
let gotToTheLowestProxy = false;
for (let i = 0; i < 20; i++) {
@@ -262,7 +261,45 @@ describe('ProxyConfiguration', () => {
gotToTheLowestProxy = true;
break;
}
- // We don't increment the sessionRotationCount here - this causes the proxy tier to go down (current proxy is ok, so it tries to downshift in some time)
+ }
+
+ expect(gotToTheLowestProxy).toBe(true);
+ });
+
+ test('successful requests make the proxy tier drop eventually', async () => {
+ const tieredProxyUrls = [['http://proxy.com:1111'], ['http://proxy.com:2222'], ['http://proxy.com:3333']];
+
+ const proxyConfiguration = new ProxyConfiguration({
+ tieredProxyUrls,
+ });
+
+ const failingRequest = new Request({
+ url: 'http://example.com',
+ });
+ let gotToTheHighestProxy = false;
+
+ for (let i = 0; i < 10; i++) {
+ const lastProxyUrl = await proxyConfiguration.newUrl('session-id', { request: failingRequest });
+
+ if (lastProxyUrl === tieredProxyUrls[2][0]) {
+ gotToTheHighestProxy = true;
+ break;
+ }
+ }
+
+ expect(gotToTheHighestProxy).toBe(true);
+
+ let gotToTheLowestProxy = false;
+
+ for (let i = 0; i < 100; i++) {
+ const lastProxyUrl = await proxyConfiguration.newUrl('session-id', {
+ request: new Request({ url: `http://example.com/${i}` }),
+ });
+
+ if (lastProxyUrl === tieredProxyUrls[0][0]) {
+ gotToTheLowestProxy = true;
+ break;
+ }
}
expect(gotToTheLowestProxy).toBe(true);