Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: use merged cookies correctly in `GotScrapingHttpClient` (#3000)

Jindřich Bära2985f259f06
diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.ts
index 45cfeb317..be75c6daf 100644
--- a/packages/core/src/http_clients/got-scraping-http-client.ts
+++ b/packages/core/src/http_clients/got-scraping-http-client.ts
@@ -23,6 +23,9 @@ export class GotScrapingHttpClient implements BaseHttpClient {
): Promise<HttpResponse<TResponseType>> {
const gotResult = await gotScraping({
...request,
+ // `HttpCrawler` reads the cookies beforehand and sets them in `request.gotOptions`.
+ // Using the `cookieJar` option directly would override that.
+ cookieJar: undefined,
retry: {
limit: 0,
...(request.retry as Record<string, unknown> | undefined),
@@ -42,7 +45,7 @@ export class GotScrapingHttpClient implements BaseHttpClient {
async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {
// eslint-disable-next-line no-async-promise-executor
return new Promise(async (resolve, reject) => {
- const stream = await Promise.resolve(gotScraping({ ...request, isStream: true }));
+ const stream = await Promise.resolve(gotScraping({ ...request, isStream: true, cookieJar: undefined }));
stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {
handleRedirect?.(redirectResponse, updatedOptions);
diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts
index 9ccb08150..d84e8014b 100644
--- a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts
+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts
@@ -44,9 +44,7 @@ class CurlImpersonateHttpClient implements BaseHttpClient {
}
if (request.cookieJar) {
- result.headers['set-cookie'] = await Promise.resolve(
- request.cookieJar.getCookieString(request.url.toString(), {}, () => {}),
- );
+ result.headers.cookie = (await (request.cookieJar as any)?.getCookieString?.(request.url.toString())) ?? '';
}
return result;