Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(http-crawler): forward the context proxy URL to the HTTP client (#4152)

Jindřich Bärb4e6f2158b6f
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 73578a3ec..1b4f09832 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -920,6 +920,7 @@ export class HttpCrawler<
{
session,
cookieJar,
+ proxyUrl: opts.proxyUrl,
signal: cancelSignal,
timeoutMillis: cancelSignal ? undefined : opts.timeout,
ignoreTlsErrors: this.#ignoreTlsErrors,
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index cd8aaedef..909285a0e 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -14,6 +14,7 @@ import {
ThrottlingRequestManager,
} from '@crawlee/http';
import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
+import type { ProxyInfo, SendRequestOptions } from '@crawlee/types';
import { sleep } from '@crawlee/utils';
import iconv from 'iconv-lite';
@@ -643,6 +644,35 @@ test('works with a custom HttpClient', async () => {
expect(results[1].includes('Schmexample Domain')).toBeTruthy();
});
+test('forwards the context proxyInfo url to the HttpClient', async () => {
+ const proxyUrls: (string | undefined)[] = [];
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ preNavigationHooks: [
+ async (context) => {
+ // You probably don't want this in real code - let `ProxyConfiguration` assign proxies via sessions.
+ context.proxyInfo = { url: 'http://proxy.example.com:8000' } as ProxyInfo;
+ },
+ ],
+ requestHandler: async () => {},
+ httpClient: Object.assign(Object.create(BaseHttpClient.prototype) as BaseHttpClient, {
+ async sendRequest(request: Request, options?: SendRequestOptions) {
+ proxyUrls.push(options?.proxyUrl);
+ return new ResponseWithUrl('<html></html>', {
+ url: request.url.toString(),
+ status: 200,
+ headers: { 'content-type': 'text/html; charset=utf-8' },
+ });
+ },
+ }),
+ });
+
+ await crawler.run([url]);
+
+ expect(proxyUrls).toEqual(['http://proxy.example.com:8000']);
+});
+
test('a 429 on a throttled domain paces the retry without spending it or the session', async () => {
const hits: number[] = [];
router.set('/429-then-ok', (req, res) => {