Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: use merged cookies correctly in `GotScrapingHttpClient` (#3000)
Jindřich Bära2985f259f06
diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.tsindex 45cfeb317..be75c6daf 100644--- a/packages/core/src/http_clients/got-scraping-http-client.ts+++ b/packages/core/src/http_clients/got-scraping-http-client.ts@@ -23,6 +23,9 @@ export class GotScrapingHttpClient implements BaseHttpClient {): Promise<HttpResponse<TResponseType>> {const gotResult = await gotScraping({...request,+ // `HttpCrawler` reads the cookies beforehand and sets them in `request.gotOptions`.+ // Using the `cookieJar` option directly would override that.+ cookieJar: undefined,retry: {limit: 0,...(request.retry as Record<string, unknown> | undefined),@@ -42,7 +45,7 @@ export class GotScrapingHttpClient implements BaseHttpClient {async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {// eslint-disable-next-line no-async-promise-executorreturn new Promise(async (resolve, reject) => {- const stream = await Promise.resolve(gotScraping({ ...request, isStream: true }));+ const stream = await Promise.resolve(gotScraping({ ...request, isStream: true, cookieJar: undefined }));stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {handleRedirect?.(redirectResponse, updatedOptions);diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.tsindex 9ccb08150..d84e8014b 100644--- a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts@@ -44,9 +44,7 @@ class CurlImpersonateHttpClient implements BaseHttpClient {}if (request.cookieJar) {- result.headers['set-cookie'] = await Promise.resolve(- request.cookieJar.getCookieString(request.url.toString(), {}, () => {}),- );+ result.headers.cookie = (await (request.cookieJar as any)?.getCookieString?.(request.url.toString())) ?? '';}return result;