Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(http-crawler): replace `IncomingMessage` with `PlainResponse` for context's `response` (#1973)

Sviatozar Petrenko2a1cc7f4f87f
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index 7fc500be9..4b5c67940 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -51,6 +51,9 @@ export enum RequestState {
* are considered as pointing to the same web resource. This behavior applies to all Crawlee classes,
* such as {@apilink RequestList}, {@apilink RequestQueue}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}.
*
+ * > To access and examine the actual request sent over http, with all autofilled headers you can access
+ * `response.request` object from the request handler
+ *
* Example use:
*
* ```javascript
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 9c8cbf914..d797a5d02 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -27,7 +27,7 @@ import type { RequestLike, ResponseLike } from 'content-type';
import * as cheerio from 'cheerio';
import contentTypeParser from 'content-type';
import mime from 'mime-types';
-import type { OptionsInit, Method, Request as GotRequest, Options } from 'got-scraping';
+import type { OptionsInit, Method, Request as GotRequest, Options, PlainResponse } from 'got-scraping';
import { gotScraping, TimeoutError } from 'got-scraping';
import type { JsonValue } from 'type-fest';
import { extname } from 'node:path';
@@ -191,7 +191,7 @@ export interface InternalHttpCrawlingContext<
* Parsed `Content-Type header: { type, encoding }`.
*/
contentType: { type: string; encoding: BufferEncoding };
- response: IncomingMessage;
+ response: PlainResponse;
parseWithCheerio(): Promise<cheerio.CheerioAPI>;
}
@@ -569,7 +569,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
* on the request such as only downloading the request body if the
* received content type matches text/html, application/xml, application/xhtml+xml.
*/
- protected async _requestFunction({ request, session, proxyUrl, gotOptions }: RequestFunctionOptions): Promise<IncomingMessage> {
+ protected async _requestFunction({ request, session, proxyUrl, gotOptions }: RequestFunctionOptions): Promise<PlainResponse> {
const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
try {
@@ -577,7 +577,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
} catch (e) {
if (e instanceof TimeoutError) {
this._handleRequestTimeout(session);
- return undefined as unknown as IncomingMessage;
+ return undefined as unknown as PlainResponse;
}
throw e;
@@ -752,10 +752,10 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
* @internal wraps public utility for mocking purposes
*/
private _requestAsBrowser = (options: OptionsInit & { isStream: true }, session?: Session) => {
- return new Promise<IncomingMessage>((resolve, reject) => {
+ return new Promise<PlainResponse>((resolve, reject) => {
const stream = gotScraping(options);
- stream.on('redirect', (updatedOptions: Options, redirectResponse: IncomingMessage) => {
+ stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {
if (this.persistCookiesPerSession) {
session!.setCookiesFromResponse(redirectResponse);
@@ -812,11 +812,11 @@ function addResponsePropertiesToStream(stream: GotRequest) {
for (const prop of properties) {
if (!(prop in stream)) {
// @ts-expect-error
- stream[prop] = response[prop as keyof IncomingMessage];
+ stream[prop] = response[prop as keyof PlainResponse];
}
}
- return stream as unknown as IncomingMessage;
+ return stream as unknown as PlainResponse;
}
/**
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index d459c8865..6021f56f7 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -380,8 +380,6 @@ describe('CheerioCrawler', () => {
const crawler = new CheerioCrawler({
requestList,
requestHandler: ({ response }) => {
- // TODO: this accesses IncomingMessage#request, which doesn't exist according to types
- // @ts-expect-error
headers.push(response.request.options.headers);
},
});