Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: patch `GotScrapingHttpClient` header handling and proxies (#3308)

Jindřich Bär57d1f43e0695
diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.ts
index a248fcc13..caf40e77d 100644
--- a/packages/core/src/http_clients/got-scraping-http-client.ts
+++ b/packages/core/src/http_clients/got-scraping-http-client.ts
@@ -20,6 +20,29 @@ export class GotScrapingHttpClient implements BaseHttpClient {
return !['CONNECT', 'connect'].includes(request.method!);
}
+ private *iterateHeaders(
+ headers: Record<string, string | string[] | undefined>,
+ ): Generator<[string, string], void, unknown> {
+ for (const [key, value] of Object.entries(headers)) {
+ // Filter out pseudo-headers
+ if (key.startsWith(':') || value === undefined) {
+ continue;
+ }
+
+ if (Array.isArray(value)) {
+ for (const v of value) {
+ yield [key, v];
+ }
+ } else {
+ yield [key, value];
+ }
+ }
+ }
+
+ private parseHeaders(headers: Record<string, string | string[] | undefined>): Headers {
+ return new Headers([...this.iterateHeaders(headers)]);
+ }
+
/**
* @inheritDoc
*/
@@ -43,20 +66,10 @@ export class GotScrapingHttpClient implements BaseHttpClient {
cookieJar: undefined,
});
- const parsedHeaders = Object.entries(gotResult.headers)
- .map(([key, value]) => {
- if (value === undefined) return [];
-
- if (Array.isArray(value)) {
- return value.map((v) => [key, v]);
- }
-
- return [[key, value]];
- })
- .flat() as [string, string][];
+ const responseHeaders = this.parseHeaders(gotResult.headers);
return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), {
- headers: new Headers(parsedHeaders),
+ headers: responseHeaders,
status: gotResult.statusCode,
statusText: gotResult.statusMessage ?? '',
url: gotResult.url,
@@ -67,6 +80,8 @@ export class GotScrapingHttpClient implements BaseHttpClient {
* @inheritDoc
*/
async stream(request: Request, options?: StreamOptions): Promise<Response> {
+ const { session, timeout } = options ?? {};
+
if (!this.validateRequest(request)) {
throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
}
@@ -78,6 +93,8 @@ export class GotScrapingHttpClient implements BaseHttpClient {
headers: Object.fromEntries(request.headers.entries()),
body: request.body ? Readable.fromWeb(request.body as any) : undefined,
isStream: true,
+ proxyUrl: session?.proxyInfo?.url,
+ timeout: { request: timeout },
cookieJar: undefined,
});
@@ -111,12 +128,13 @@ export class GotScrapingHttpClient implements BaseHttpClient {
stream.on('error', reject);
stream.on('response', (response: PlainResponse) => {
+ const headers = this.parseHeaders(response.headers);
// Cast shouldn't be needed here, undici might have a different `ReadableStream` type
resolve(
new ResponseWithUrl(Readable.toWeb(stream) as any, {
status: response.statusCode,
statusText: response.statusMessage ?? '',
- headers: response.headers as HeadersInit,
+ headers,
url: response.url,
}),
);
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 2801a1bd5..6f3017608 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -690,7 +690,7 @@ export class HttpCrawler<
url: request.url,
method: request.method,
proxyUrl,
- timeout: { request: this.navigationTimeoutMillis },
+ timeout: this.navigationTimeoutMillis,
cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
sessionToken: session,
headers: request.headers,