Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: `sendRequest` uses correct `Session` instance and allows for overrides (#3313)
Jindřich Bärdb31875811e0
diff --git a/packages/basic-crawler/src/internals/send-request.ts b/packages/basic-crawler/src/internals/send-request.tsindex e8d8b5608..344e63a38 100644--- a/packages/basic-crawler/src/internals/send-request.ts+++ b/packages/basic-crawler/src/internals/send-request.ts@@ -1,5 +1,5 @@-import type { Request, Session } from '@crawlee/core';-import type { BaseHttpClient, HttpRequestOptions } from '@crawlee/types';+import type { Request as CrawleeRequest, Session } from '@crawlee/core';+import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@crawlee/types';/*** Prepares a function to be used as the `sendRequest` context helper.@@ -8,18 +8,42 @@ import type { BaseHttpClient, HttpRequestOptions } from '@crawlee/types';* @param httpClient The HTTP client that will perform the requests.* @param originRequest The crawling request being processed.* @param session The user session associated with the current request.- * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.*/-export function createSendRequest(httpClient: BaseHttpClient, originRequest: Request, session: Session | undefined) {- return async (overrideOptions: Partial<HttpRequestOptions> = {}): Promise<Response> => {- const cookieJar = session- ? {- getCookieString: async (url: string) => session.getCookieString(url),- setCookie: async (rawCookie: string, url: string) => session.setCookie(rawCookie, url),- ...overrideOptions?.cookieJar,- }- : overrideOptions?.cookieJar;+export function createSendRequest(+ httpClient: BaseHttpClient,+ originRequest: CrawleeRequest,+ session: Session | undefined,+) {+ return async (+ overrideRequest: Partial<HttpRequestOptions> = {},+ overrideOptions: SendRequestOptions = {},+ ): Promise<Response> => {+ const sessionCookieJar = {+ getCookieString: async (url: string) => session?.getCookieString(url),+ setCookie: async (rawCookie: string, url: string) => session?.setCookie(rawCookie, url),+ };- return httpClient.sendRequest(originRequest.intoFetchAPIRequest(), { session, cookieJar: cookieJar as any });+ const baseRequest = originRequest.intoFetchAPIRequest();+ const mergedUrl = overrideRequest.url ?? baseRequest.url;+ const mergedMethod = overrideRequest.method ?? baseRequest.method;++ const mergedHeaders = new Headers(baseRequest.headers);+ if (overrideRequest.headers) {+ overrideRequest.headers.forEach((value, key) => {+ mergedHeaders.set(key, value);+ });+ }++ const request = new Request(mergedUrl, {+ method: mergedMethod,+ headers: mergedHeaders,+ body: overrideRequest.body ?? baseRequest.body,+ } as RequestInit);++ return httpClient.sendRequest(request, {+ session,+ cookieJar: overrideOptions?.cookieJar ?? (sessionCookieJar as any),+ timeout: overrideOptions.timeout,+ });};}diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.tsindex f0c9c4e11..6e88d3b02 100644--- a/packages/core/src/http_clients/got-scraping-http-client.ts+++ b/packages/core/src/http_clients/got-scraping-http-client.ts@@ -47,7 +47,7 @@ export class GotScrapingHttpClient implements BaseHttpClient {* @inheritDoc*/async sendRequest(request: Request, options?: SendRequestOptions): Promise<Response> {- const { session, timeout, proxyUrl } = options ?? {};+ const { session, timeout, cookieJar, proxyUrl } = options ?? {};if (!this.validateRequest(request)) {throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);@@ -60,10 +60,7 @@ export class GotScrapingHttpClient implements BaseHttpClient {body: request.body ? Readable.fromWeb(request.body as any) : undefined,proxyUrl: proxyUrl ?? session?.proxyInfo?.url,timeout: { request: timeout },- // We set cookieJar to undefined because- // `HttpCrawler` reads the cookies beforehand and sets them in `request.headers`.- // Using the `cookieJar` option directly would override that.- cookieJar: undefined,+ cookieJar,});const responseHeaders = this.parseHeaders(gotResult.headers);diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 8e68dc574..a41909427 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1158,6 +1158,45 @@ describe('CheerioCrawler', () => {'Could not parse cookie header string: sessionid',);});++ test('sendRequest and main request should share the same session cookie jar', async () => {+ const responses: { cookies: string }[] = [];++ const crawler = new CheerioCrawler({+ requestList: await RequestList.open(null, [{ url: `${serverAddress}/special/get-cookies` }]),+ useSessionPool: true,+ sessionPoolOptions: {+ // Even with multiple available sessions, the preNavigationHook should use the same one as the main request+ maxPoolSize: 10,+ },+ preNavigationHooks: [+ async ({ sendRequest }) => {+ await sendRequest({+ url: `${serverAddress}/special/set-cookie?name=sharedCookie&value=sharedValue`,+ });++ const response = await sendRequest({ url: `${serverAddress}/special/get-cookies` });+ const json = await response.json();++ expect(json.cookies).toContain('sharedCookie=sharedValue');+ },+ ],+ requestHandler: async ({ json, sendRequest }) => {+ responses.push(json as { cookies: string });++ const sendRequestJson = await sendRequest({ url: `${serverAddress}/special/get-cookies` }).then(+ async (response) => response.json(),+ );+ responses.push(sendRequestJson as { cookies: string });+ },+ });++ await crawler.run();++ expect(responses).toHaveLength(2);+ expect(responses[0].cookies).toContain('sharedCookie=sharedValue');+ expect(responses[1].cookies).toContain('sharedCookie=sharedValue');+ });});describe('Crawling context', () => {diff --git a/test/shared/_helper.ts b/test/shared/_helper.tsindex 35a137edd..45ca9ad20 100644--- a/test/shared/_helper.ts+++ b/test/shared/_helper.ts@@ -336,6 +336,17 @@ export async function runExampleComServer(): Promise<[Server, number]> {special.get('/html-entities', (_req, res) => {res.type('html').send('"<>"<>');});++ special.get('/set-cookie', (req, res) => {+ const cookieName = (req.query.name as string) || 'testCookie';+ const cookieValue = (req.query.value as string) || 'testValue';+ res.setHeader('set-cookie', `${cookieName}=${cookieValue}; Path=/`);+ res.type('html').send('<html><body>Cookie set</body></html>');+ });++ special.get('/get-cookies', (req, res) => {+ res.json({ cookies: req.headers.cookie || '' });+ });})();// "cacheable" site with one page, scripts and stylesheets