Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: sendRequest types (#1445)
Szymon Marczak751563b4f692
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex adbfef1d8..af4a4cd2e 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -34,7 +34,7 @@ import {purgeDefaultStorages,validators,} from '@crawlee/core';-import type { GotOptionsInit, OptionsOfTextResponseBody, Response as GotResponse } from 'got-scraping';+import type { Method, OptionsInit, Response as GotResponse } from 'got-scraping';import { gotScraping } from 'got-scraping';import type { ProcessedRequest, Dictionary, Awaitable, BatchAddRequestsResult } from '@crawlee/types';import { chunk, sleep } from '@crawlee/utils';@@ -43,7 +43,7 @@ import ow, { ArgumentError } from 'ow';export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {crawler: BasicCrawler;enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;- sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;+ sendRequest: (overrideOptions?: Partial<OptionsInit>) => Promise<GotResponse<string>>;}/** @internal */@@ -829,10 +829,16 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextrequestQueue: await this.getRequestQueue(),});},- sendRequest: async (overrideOptions?: GotOptionsInit) => {+ sendRequest: async (overrideOptions?: OptionsInit) => {+ const cookieJar = session ? {+ getCookieString: async (url: string) => session!.getCookieString(url),+ setCookie: async (rawCookie: string, url: string) => session!.setCookie(rawCookie, url),+ ...overrideOptions?.cookieJar,+ } : overrideOptions?.cookieJar;+return gotScraping({url: request!.url,- method: request!.method,+ method: request!.method as Method, // Narrow type to omit CONNECTbody: request!.payload,headers: request!.headers,proxyUrl: crawlingContext.proxyInfo?.url,@@ -843,12 +849,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextlimit: 0,...overrideOptions?.retry,},- cookieJar: {- getCookieString: (url: string) => session!.getCookieString(url),- setCookie: (rawCookie: string, url: string) => session!.setCookie(rawCookie, url),- ...overrideOptions?.cookieJar,- },- } as OptionsOfTextResponseBody);+ cookieJar,+ });},};diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex aac59d04d..d48cc106a 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -1051,9 +1051,11 @@ describe('BasicCrawler', () => {});describe('sendRequest', () => {+ const html = `<!DOCTYPE html><html><head><title>foobar</title></head><body><p>Hello, world!</p></body></html>`;+const httpServer = http.createServer((request, response) => {response.setHeader('content-type', 'text/html');- response.end(`<!DOCTYPE html><html><head><title>foobar</title></head><body><p>Hello, world!</p></body></html>`);+ response.end(html);});let url: string;@@ -1071,21 +1073,72 @@ describe('BasicCrawler', () => {});test('works', async () => {- expect.assertions(2);+ const responses: { statusCode: number; body: string }[] = [];++ const requestList = await RequestList.open(null, [url]);++ const crawler = new BasicCrawler({+ useSessionPool: true,+ requestList,+ async requestHandler({ sendRequest }) {+ const response = await sendRequest();++ responses.push({+ statusCode: response.statusCode,+ body: response.body,+ });+ },+ });++ await crawler.run();+ expect(responses).toStrictEqual([+ {+ statusCode: 200,+ body: html,+ },+ ]);+ });++ test('works without session', async () => {const requestList = await RequestList.open(null, [url]);+ const responses: { statusCode: number; body: string }[] = [];+const crawler = new BasicCrawler({+ useSessionPool: false,requestList,async requestHandler({ sendRequest }) {const response = await sendRequest();- expect(response.statusCode).toBe(200);- expect(response.body.includes('Hello, world!')).toBe(true);+ responses.push({+ statusCode: response.statusCode,+ body: response.body,+ });},});await crawler.run();++ expect(responses).toStrictEqual([+ {+ statusCode: 200,+ body: html,+ },+ ]);+ });++ test('proxyUrl TypeScript support', async () => {+ const crawler = new BasicCrawler({+ useSessionPool: true,+ async requestHandler({ sendRequest }) {+ await sendRequest({+ proxyUrl: 'http://example.com',+ });+ },+ });++ expect(crawler).toBeTruthy();});});});