Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: allow using other HTTP clients (#2661)
Jan Buchar568c6556d79c
diff --git a/docs/guides/custom-http-client/custom-http-client.mdx b/docs/guides/custom-http-client/custom-http-client.mdxnew file mode 100644index 000000000..c593ec3ba--- /dev/null+++ b/docs/guides/custom-http-client/custom-http-client.mdx@@ -0,0 +1,23 @@+---+id: custom-http-client+title: Using a custom HTTP client (Experimental)+description: Use a custom HTTP client for `sendRequest` and plain-HTTP crawling+---++import ApiLink from '@site/src/components/ApiLink';+import CodeBlock from '@theme/CodeBlock';++import ImplementationSource from '!!raw-loader!./implementation.ts';+import UsageSource from '!!raw-loader!./usage.ts';++The <ApiLink to="basic-crawler/class/BasicCrawler">`BasicCrawler`</ApiLink> class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate` or `axios`.++The HTTP client implementation needs to conform to the <ApiLink to="core/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> interface. For a rough idea on how it might look, see a skeleton implementation that uses the standard `fetch` interface:++<CodeBlock language="ts">{ImplementationSource}</CodeBlock>++You may then instantiate it and pass to a crawler constructor:++<CodeBlock language="ts">{UsageSource}</CodeBlock>++Please note that the interface is experimental and it will likely change with Crawlee version 4.diff --git a/docs/guides/custom-http-client/implementation.ts b/docs/guides/custom-http-client/implementation.tsnew file mode 100644index 000000000..ba86b6eae--- /dev/null+++ b/docs/guides/custom-http-client/implementation.ts@@ -0,0 +1,122 @@+import {+ BaseHttpClient,+ HttpRequest,+ HttpResponse,+ RedirectHandler,+ ResponseTypes,+ StreamingHttpResponse,+} from '@crawlee/core';+import { Readable } from 'node:stream';++class CustomHttpClient implements BaseHttpClient {+ async sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(+ request: HttpRequest<TResponseType>,+ ): Promise<HttpResponse<TResponseType>> {+ const requestHeaders = new Headers();+ for (let [headerName, headerValues] of Object.entries(request.headers ?? {})) {+ if (headerValues === undefined) {+ continue;+ }++ if (!Array.isArray(headerValues)) {+ headerValues = [headerValues];+ }++ for (const value of headerValues) {+ requestHeaders.append(headerName, value);+ }+ }++ const response = await fetch(request.url, {+ method: request.method,+ headers: requestHeaders,+ body: request.body as string, // TODO implement stream/generator handling+ signal: request.signal,+ // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)+ });++ const headers: Record<string, string> = {};++ response.headers.forEach((value, headerName) => {+ headers[headerName] = value;+ });++ return {+ complete: true,+ request,+ url: response.url,+ statusCode: response.status,+ redirectUrls: [], // TODO you need to handle redirects manually to track them+ headers,+ trailers: {}, // TODO not supported by fetch+ ip: undefined,+ body:+ request.responseType === 'text'+ ? await response.text()+ : request.responseType === 'json'+ ? await response.json()+ : Buffer.from(await response.text()),+ };+ }++ async stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {+ const fetchResponse = await fetch(request.url, {+ method: request.method,+ headers: new Headers(),+ body: request.body as string, // TODO implement stream/generator handling+ signal: request.signal,+ // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)+ });++ const headers: Record<string, string> = {}; // TODO same as in sendRequest()++ async function* read() {+ const reader = fetchResponse.body?.getReader();++ const stream = new ReadableStream({+ start(controller) {+ if (!reader) {+ return null;+ }+ return pump();+ function pump() {+ return reader!.read().then(({ done, value }) => {+ // When no more data needs to be consumed, close the stream+ if (done) {+ controller.close();+ return;+ }+ // Enqueue the next data chunk into our target stream+ controller.enqueue(value);+ return pump();+ });+ }+ },+ });++ for await (const chunk of stream) {+ yield chunk;+ }+ }++ const response = {+ complete: false,+ request,+ url: fetchResponse.url,+ statusCode: fetchResponse.status,+ redirectUrls: [], // TODO you need to handle redirects manually to track them+ headers,+ trailers: {}, // TODO not supported by fetch+ ip: undefined,+ stream: Readable.from(read()),+ get downloadProgress() {+ return { percent: 0, transferred: 0 }; // TODO track this+ },+ get uploadProgress() {+ return { percent: 0, transferred: 0 }; // TODO track this+ },+ };++ return response;+ }+}diff --git a/docs/guides/custom-http-client/usage.ts b/docs/guides/custom-http-client/usage.tsnew file mode 100644index 000000000..02e0ae919--- /dev/null+++ b/docs/guides/custom-http-client/usage.ts@@ -0,0 +1,6 @@+const crawler = new HttpCrawler({+ httpClient: new CustomHttpClient(),+ async requestHandler() {+ /* ... */+ },+});diff --git a/package.json b/package.jsonindex 9391d4b36..acc374b9c 100644--- a/package.json+++ b/package.json@@ -85,6 +85,7 @@"@typescript-eslint/parser": "^7.18.0","@vitest/coverage-v8": "^2.0.0","apify": "*",+ "apify-node-curl-impersonate": "^1.0.15","basic-auth-parser": "^0.0.2","body-parser": "^1.20.0","commitlint": "^19.0.0",diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex ae94d8590..c4c6c0ca9 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -26,6 +26,7 @@ import type {StatisticState,StatisticsOptions,LoadedContext,+ BaseHttpClient,RestrictedCrawlingContext,} from '@crawlee/core';import {@@ -50,17 +51,20 @@ import {SessionPool,Statistics,validators,+ GotScrapingHttpClient,} from '@crawlee/core';import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';-import { ROTATE_PROXY_ERRORS, gotScraping } from '@crawlee/utils';+import { ROTATE_PROXY_ERRORS } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeFile, writeJSON } from 'fs-extra';// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood-import type { OptionsInit, Method } from 'got-scraping';+import type { OptionsInit, Method, GotResponse } from 'got-scraping';import ow, { ArgumentError } from 'ow';import { getDomain } from 'tldts';import type { SetRequired } from 'type-fest';+import { createSendRequest } from './send-request';+export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>extends CrawlingContext<BasicCrawler, UserData> {/**@@ -351,6 +355,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw* whether to output them to the Key-Value store.*/statisticsOptions?: StatisticsOptions;++ /**+ * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.+ * Defaults to a new instance of {@apilink GotScrapingHttpClient}+ */+ httpClient?: BaseHttpClient;}/**@@ -496,6 +506,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextprotected crawlingContexts = new Map<string, Context>();protected autoscaledPoolOptions: AutoscaledPoolOptions;protected events: EventManager;+ protected httpClient: BaseHttpClient;protected retryOnBlocked: boolean;private _closeEvents?: boolean;@@ -530,6 +541,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextstatusMessageCallback: ow.optional.function,retryOnBlocked: ow.optional.boolean,+ httpClient: ow.optional.object,// AutoscaledPool shorthandsminConcurrency: ow.optional.number,@@ -592,10 +604,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextstatusMessageCallback,statisticsOptions,+ httpClient,} = options;this.requestList = requestList;this.requestQueue = requestQueue;+ this.httpClient = httpClient ?? new GotScrapingHttpClient();this.log = log;this.statusMessageLoggingInterval = statusMessageLoggingInterval;this.statusMessageCallback = statusMessageCallback as StatusMessageCallback;@@ -1273,31 +1287,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextaddRequests: this.addRequests.bind(this),pushData: this.pushData.bind(this),useState: this.useState.bind(this),- sendRequest: async (overrideOptions?: OptionsInit) => {- const cookieJar = session- ? {- getCookieString: async (url: string) => session!.getCookieString(url),- setCookie: async (rawCookie: string, url: string) => session!.setCookie(rawCookie, url),- ...overrideOptions?.cookieJar,- }- : overrideOptions?.cookieJar;-- return gotScraping({- url: request!.url,- method: request!.method as Method, // Narrow type to omit CONNECT- body: request!.payload,- headers: request!.headers,- proxyUrl: crawlingContext.proxyInfo?.url,- sessionToken: session,- responseType: 'text',- ...overrideOptions,- retry: {- limit: 0,- ...overrideOptions?.retry,- },- cookieJar,- });- },+ sendRequest: createSendRequest(this.httpClient, request!, session, () => crawlingContext.proxyInfo?.url),getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }),};diff --git a/packages/basic-crawler/src/internals/send-request.ts b/packages/basic-crawler/src/internals/send-request.tsnew file mode 100644index 000000000..a841efcae--- /dev/null+++ b/packages/basic-crawler/src/internals/send-request.ts@@ -0,0 +1,54 @@+import {+ type Session,+ type Request,+ type BaseHttpClient,+ type HttpRequestOptions,+ processHttpRequestOptions,+} from '@crawlee/core';+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood+import type { Method, GotResponse } from 'got-scraping';++/**+ * Prepares a function to be used as the `sendRequest` context helper.+ *+ * @internal+ * @param httpClient The HTTP client that will perform the requests.+ * @param originRequest The crawling request being processed.+ * @param session The user session associated with the current request.+ * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.+ */+export function createSendRequest(+ httpClient: BaseHttpClient,+ originRequest: Request,+ session: Session | undefined,+ getProxyUrl: () => string | undefined,+) {+ return async <Response = string>(+ // TODO the type information here (and in crawler_commons) is outright wrong... for BC - replace this with generic HttpResponse in v4+ overrideOptions: Partial<HttpRequestOptions> = {},+ ): Promise<GotResponse<Response>> => {+ const cookieJar = session+ ? {+ getCookieString: async (url: string) => session.getCookieString(url),+ setCookie: async (rawCookie: string, url: string) => session.setCookie(rawCookie, url),+ ...overrideOptions?.cookieJar,+ }+ : overrideOptions?.cookieJar;++ const requestOptions = processHttpRequestOptions({+ url: originRequest.url,+ method: originRequest.method as Method, // Narrow type to omit CONNECT+ headers: originRequest.headers,+ proxyUrl: getProxyUrl(),+ sessionToken: session,+ responseType: 'text',+ ...overrideOptions,+ cookieJar,+ });++ // Fill in body as the last step - `processHttpRequestOptions` may use either `body`, `json` or `form` so we cannot override it beforehand+ requestOptions.body ??= originRequest.payload;++ return httpClient.sendRequest<any>(requestOptions);+ };+}diff --git a/packages/browser-pool/src/fingerprinting/hooks.ts b/packages/browser-pool/src/fingerprinting/hooks.tsindex 63cab8052..898f68aa1 100644--- a/packages/browser-pool/src/fingerprinting/hooks.ts+++ b/packages/browser-pool/src/fingerprinting/hooks.ts@@ -1,9 +1,9 @@import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';import type { FingerprintInjector } from 'fingerprint-injector';-import type { BrowserPool } from '../browser-pool';import { getGeneratorDefaultOptions } from './utils';import type { BrowserController } from '../abstract-classes/browser-controller';+import type { BrowserPool } from '../browser-pool';import type { LaunchContext } from '../launch-context';import { PlaywrightPlugin } from '../playwright/playwright-plugin';import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin';diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.tsindex 950f045a0..a97477ce7 100644--- a/packages/core/src/cookie_utils.ts+++ b/packages/core/src/cookie_utils.ts@@ -1,19 +1,20 @@-import type { IncomingMessage } from 'node:http';--import type { BrowserLikeResponse, Dictionary, Cookie as CookieObject } from '@crawlee/types';+import type { Cookie as CookieObject } from '@crawlee/types';import { Cookie, CookieJar } from 'tough-cookie';import { log } from './log';import { CookieParseError } from './session_pool/errors';+export interface ResponseLike {+ url?: string | (() => string);+ headers?: Record<string, string | string[] | undefined> | (() => Record<string, string | string[] | undefined>);+}+/*** @internal*/-export function getCookiesFromResponse(- response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> },-): Cookie[] {+export function getCookiesFromResponse(response: ResponseLike): Cookie[] {const headers = typeof response.headers === 'function' ? response.headers() : response.headers;- const cookieHeader = headers['set-cookie'] || '';+ const cookieHeader = headers?.['set-cookie'] || '';try {return Array.isArray(cookieHeader)diff --git a/packages/core/src/http_clients/base-http-client.ts b/packages/core/src/http_clients/base-http-client.tsnew file mode 100644index 000000000..94491c27f--- /dev/null+++ b/packages/core/src/http_clients/base-http-client.ts@@ -0,0 +1,239 @@+import type { Readable } from 'node:stream';++import { applySearchParams, type SearchParams } from '@crawlee/utils';++import type { FormDataLike } from './form-data-like';++type Timeout =+ | {+ lookup: number;+ connect: number;+ secureConnect: number;+ socket: number;+ send: number;+ response: number;+ }+ | { request: number };++type Method =+ | 'GET'+ | 'POST'+ | 'PUT'+ | 'PATCH'+ | 'HEAD'+ | 'DELETE'+ | 'OPTIONS'+ | 'TRACE'+ | 'get'+ | 'post'+ | 'put'+ | 'patch'+ | 'head'+ | 'delete'+ | 'options'+ | 'trace';++/**+ * Maps permitted values of the `responseType` option on {@apilink HttpRequest} to the types that they produce.+ */+export interface ResponseTypes {+ 'json': unknown;+ 'text': string;+ 'buffer': Buffer;+}++interface Progress {+ percent: number;+ transferred: number;+ total?: number;+}++// TODO BC with got - remove the options and callback parameters in 4.0+interface ToughCookieJar {+ getCookieString: ((+ currentUrl: string,+ options: Record<string, unknown>,+ callback: (error: Error | null, cookies: string) => void,+ ) => string) &+ ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string);+ setCookie: ((+ cookieOrString: unknown,+ currentUrl: string,+ options: Record<string, unknown>,+ callback: (error: Error | null, cookie: unknown) => void,+ ) => void) &+ ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void);+}++interface PromiseCookieJar {+ getCookieString: (url: string) => Promise<string>;+ setCookie: (rawCookie: string, url: string) => Promise<unknown>;+}++type SimpleHeaders = Record<string, string | string[] | undefined>;++/**+ * HTTP Request as accepted by {@apilink BaseHttpClient} methods.+ */+export interface HttpRequest<TResponseType extends keyof ResponseTypes = 'text'> {+ [k: string]: unknown; // TODO BC with got - remove in 4.0++ url: string | URL;+ method?: Method;+ headers?: SimpleHeaders;+ body?: string | Buffer | Readable | Generator | AsyncGenerator | FormDataLike;++ signal?: AbortSignal;+ timeout?: Partial<Timeout>;++ cookieJar?: ToughCookieJar | PromiseCookieJar;+ followRedirect?: boolean | ((response: any) => boolean); // TODO BC with got - specify type better in 4.0+ maxRedirects?: number;++ encoding?: BufferEncoding;+ responseType?: TResponseType;+ throwHttpErrors?: boolean;++ // from got-scraping Context+ proxyUrl?: string;+ headerGeneratorOptions?: Record<string, unknown>;+ useHeaderGenerator?: boolean;+ headerGenerator?: {+ getHeaders: (options: Record<string, unknown>) => Record<string, string>;+ };+ insecureHTTPParser?: boolean;+ sessionToken?: object;+}++/**+ * Additional options for HTTP requests that need to be handled separately before passing to {@apilink BaseHttpClient}.+ */+export interface HttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'>+ extends HttpRequest<TResponseType> {+ /** Search (query string) parameters to be appended to the request URL */+ searchParams?: SearchParams;++ /** A form to be sent in the HTTP request body (URL encoding will be used) */+ form?: Record<string, string>;+ /** Artbitrary object to be JSON-serialized and sent as the HTTP request body */+ json?: unknown;++ /** Basic HTTP Auth username */+ username?: string;+ /** Basic HTTP Auth password */+ password?: string;+}++/**+ * HTTP response data, without a body, as returned by {@apilink BaseHttpClient} methods.+ */+export interface BaseHttpResponseData {+ redirectUrls: URL[];+ url: string;++ ip?: string;+ statusCode: number;+ statusMessage?: string;++ headers: SimpleHeaders;+ trailers: SimpleHeaders; // Populated after the whole message is processed++ complete: boolean;+}++interface HttpResponseWithoutBody<TResponseType extends keyof ResponseTypes = keyof ResponseTypes>+ extends BaseHttpResponseData {+ request: HttpRequest<TResponseType>;+}++/**+ * HTTP response data as returned by the {@apilink BaseHttpClient.sendRequest} method.+ */+export interface HttpResponse<TResponseType extends keyof ResponseTypes = keyof ResponseTypes>+ extends HttpResponseWithoutBody<TResponseType> {+ [k: string]: any; // TODO BC with got - remove in 4.0++ body: ResponseTypes[TResponseType];+}++/**+ * HTTP response data as returned by the {@apilink BaseHttpClient.stream} method.+ */+export interface StreamingHttpResponse extends HttpResponseWithoutBody {+ stream: Readable;+ readonly downloadProgress: Progress;+ readonly uploadProgress: Progress;+}++/**+ * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.+ */+export type RedirectHandler = (+ redirectResponse: BaseHttpResponseData,+ updatedRequest: { url?: string | URL; headers: SimpleHeaders },+) => void;++/**+ * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.+ */+export interface BaseHttpClient {+ /**+ * Perform an HTTP Request and return the complete response.+ */+ sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(+ request: HttpRequest<TResponseType>,+ ): Promise<HttpResponse<TResponseType>>;++ /**+ * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.+ */+ stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;+}++/**+ * Converts {@apilink HttpRequestOptions} to a {@apilink HttpRequest}.+ */+export function processHttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'>({+ searchParams,+ form,+ json,+ username,+ password,+ ...request+}: HttpRequestOptions<TResponseType>): HttpRequest<TResponseType> {+ const url = new URL(request.url);+ const headers = { ...request.headers };++ applySearchParams(url, searchParams);++ if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {+ throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');+ }++ const body = (() => {+ if (form !== undefined) {+ return new URLSearchParams(form).toString();+ }++ if (json !== undefined) {+ return JSON.stringify(json);+ }++ return request.body;+ })();++ if (form !== undefined) {+ headers['content-type'] ??= 'application/x-www-form-urlencoded';+ }++ if (json !== undefined) {+ headers['content-type'] ??= 'application/json';+ }++ if (username !== undefined || password !== undefined) {+ const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');+ headers.authorization = `Basic ${encodedAuth}`;+ }++ return { ...request, body, url, headers };+}diff --git a/packages/core/src/http_clients/form-data-like.ts b/packages/core/src/http_clients/form-data-like.tsnew file mode 100644index 000000000..784bd960a--- /dev/null+++ b/packages/core/src/http_clients/form-data-like.ts@@ -0,0 +1,67 @@+/**+ * This is copied from https://github.com/octet-stream/form-data-encoder+ */++interface FileLike {+ /**+ * Name of the file referenced by the File object.+ */+ readonly name: string;+ /**+ * Returns the media type ([`MIME`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types)) of the file represented by a `File` object.+ */+ readonly type: string;+ /**+ * Size of the file parts in bytes+ */+ readonly size: number;+ /**+ * The last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). Files without a known last modified date return the current date.+ */+ readonly lastModified: number;+ /**+ * Returns a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) which upon reading returns the data contained within the [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File).+ */+ stream(): ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;+ readonly [Symbol.toStringTag]?: string;+}++/**+ * A `string` or `File` that represents a single value from a set of `FormData` key-value pairs.+ */+type FormDataEntryValue = string | FileLike;+/**+ * This interface reflects minimal shape of the FormData+ */+export interface FormDataLike {+ /**+ * Appends a new value onto an existing key inside a FormData object,+ * or adds the key if it does not already exist.+ *+ * The difference between `set()` and `append()` is that if the specified key already exists, `set()` will overwrite all existing values with the new one, whereas `append()` will append the new value onto the end of the existing set of values.+ *+ * @param name The name of the field whose data is contained in `value`.+ * @param value The field's value. This can be [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob)+ or [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File). If none of these are specified the value is converted to a string.+ * @param fileName The filename reported to the server, when a Blob or File is passed as the second parameter. The default filename for Blob objects is "blob". The default filename for File objects is the file's filename.+ */+ append(name: string, value: unknown, fileName?: string): void;+ /**+ * Returns all the values associated with a given key from within a `FormData` object.+ *+ * @param {string} name A name of the value you want to retrieve.+ *+ * @returns An array of `FormDataEntryValue` whose key matches the value passed in the `name` parameter. If the key doesn't exist, the method returns an empty list.+ */+ getAll(name: string): FormDataEntryValue[];+ /**+ * Returns an [`iterator`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols) allowing to go through the `FormData` key/value pairs.+ * The key of each pair is a string; the value is a [`FormDataValue`](https://developer.mozilla.org/en-US/docs/Web/API/FormDataEntryValue).+ */+ entries(): IterableIterator<[string, FormDataEntryValue]>;+ /**+ * An alias for FormDataLike#entries()+ */+ [Symbol.iterator](): IterableIterator<[string, FormDataEntryValue]>;+ readonly [Symbol.toStringTag]?: string;+}diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.tsnew file mode 100644index 000000000..e9ef09079--- /dev/null+++ b/packages/core/src/http_clients/got-scraping-http-client.ts@@ -0,0 +1,93 @@+import { gotScraping } from '@crawlee/utils';+// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood+import type { Options, PlainResponse } from 'got-scraping';++import type {+ HttpRequest,+ HttpResponse,+ RedirectHandler,+ ResponseTypes,+ StreamingHttpResponse,+ BaseHttpClient,+} from './base-http-client';++/**+ * A HTTP client implementation based on the `got-scraping` library.+ */+export class GotScrapingHttpClient implements BaseHttpClient {+ /**+ * @inheritDoc+ */+ async sendRequest<TResponseType extends keyof ResponseTypes>(+ request: HttpRequest<TResponseType>,+ ): Promise<HttpResponse<TResponseType>> {+ const gotResult = await gotScraping({+ ...request,+ retry: {+ limit: 0,+ ...(request.retry as Record<string, unknown> | undefined),+ },+ });++ return {+ ...gotResult,+ body: gotResult.body as ResponseTypes[TResponseType],+ request: { url: request.url, ...gotResult.request },+ };+ }++ /**+ * @inheritDoc+ */+ async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {+ // eslint-disable-next-line no-async-promise-executor+ return new Promise(async (resolve, reject) => {+ const stream = await Promise.resolve(gotScraping({ ...request, isStream: true }));++ stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {+ handleRedirect?.(redirectResponse, updatedOptions);+ });++ // We need to end the stream for DELETE requests, otherwise it will hang.+ if (request.method && ['DELETE', 'delete'].includes(request.method)) {+ stream.end();+ }++ stream.on('error', reject);++ stream.on('response', (response: PlainResponse) => {+ const result: StreamingHttpResponse = {+ stream,+ request,+ redirectUrls: response.redirectUrls,+ url: response.url,+ ip: response.ip,+ statusCode: response.statusCode,+ headers: response.headers,+ trailers: response.trailers,+ complete: response.complete,+ get downloadProgress() {+ return stream.downloadProgress;+ },+ get uploadProgress() {+ return stream.uploadProgress;+ },+ };++ Object.assign(result, response); // TODO BC - remove in 4.0++ resolve(result);++ stream.on('end', () => {+ result.complete = response.complete;++ result.trailers ??= {};+ Object.assign(result.trailers, response.trailers);++ (result as any).rawTrailers ??= []; // TODO BC - remove in 4.0+ Object.assign((result as any).rawTrailers, response.rawTrailers);+ });+ });+ });+ }+}diff --git a/packages/core/src/http_clients/index.ts b/packages/core/src/http_clients/index.tsnew file mode 100644index 000000000..58c1b27a5--- /dev/null+++ b/packages/core/src/http_clients/index.ts@@ -0,0 +1,2 @@+export * from './base-http-client';+export * from './got-scraping-http-client';diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 4d89bec88..affb1d678 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -4,6 +4,7 @@ export * from './configuration';export * from './crawlers';export * from './enqueue_links';export * from './events';+export * from './http_clients';export * from './log';export * from './proxy_configuration';export * from './request';diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex 98d5ee1cb..8154cf012 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -1,14 +1,14 @@import { EventEmitter } from 'node:events';-import type { IncomingMessage } from 'node:http';import type { Log } from '@apify/log';import { cryptoRandomObjectId } from '@apify/utilities';-import type { BrowserLikeResponse, Cookie as CookieObject, Dictionary } from '@crawlee/types';+import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';import ow from 'ow';import type { Cookie, SerializedCookieJar } from 'tough-cookie';import { CookieJar } from 'tough-cookie';import { EVENT_SESSION_RETIRED } from './events';+import type { ResponseLike } from '../cookie_utils';import {browserPoolCookieToToughCookie,getCookiesFromResponse,@@ -305,9 +305,7 @@ export class Session {** It then parses and saves the cookies from the `set-cookie` header, if available.*/- setCookiesFromResponse(- response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]>; url: string },- ) {+ setCookiesFromResponse(response: ResponseLike) {try {const cookies = getCookiesFromResponse(response).filter((c) => c);this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url!);diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex e5badc559..b816c8c4a 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -25,7 +25,7 @@ export type StreamHandlerContext = Omit<FileDownloadCrawlingContext,'body' | 'response' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'> & {- stream: Request;+ stream: Request; // TODO BC - remove in v4};type StreamHandler = (context: StreamHandlerContext) => void | Promise<void>;@@ -139,32 +139,29 @@ export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {request: { url },} = context;- const { gotScraping } = await import('got-scraping');-- const stream = gotScraping.stream({+ const response = await this.httpClient.stream({url,timeout: { request: undefined },proxyUrl: context.proxyInfo?.url,- isStream: true,});let pollingInterval: NodeJS.Timeout | undefined;const cleanUp = () => {clearInterval(pollingInterval!);- stream.destroy();+ response.stream.destroy();};const downloadPromise = new Promise<void>((resolve, reject) => {pollingInterval = setInterval(() => {- const { total, transferred } = stream.downloadProgress;+ const { total, transferred } = response.downloadProgress;if (transferred > 0) {log.debug(`Downloaded ${transferred} bytes of ${total ?? 0} bytes from ${url}.`);}}, 5000);- stream.on('error', async (error: Error) => {+ response.stream.on('error', async (error: Error) => {cleanUp();reject(error);});@@ -172,7 +169,7 @@ export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {let streamHandlerResult;try {- context.stream = stream;+ context.stream = response.stream;streamHandlerResult = this.streamHandler!(context as any);} catch (e) {cleanUp();@@ -193,7 +190,7 @@ export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {}});- await Promise.all([downloadPromise, finished(stream)]);+ await Promise.all([downloadPromise, finished(response.stream)]);cleanUp();}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 883fb3bec..30b1bc6f5 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -1,6 +1,7 @@import type { IncomingHttpHeaders, IncomingMessage } from 'node:http';import { extname } from 'node:path';import util from 'node:util';+import type { Readable } from 'stream';import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';@@ -27,28 +28,36 @@ import {Configuration,RequestState,SessionError,+ processHttpRequestOptions,} from '@crawlee/basic';+import type { HttpResponse, StreamingHttpResponse } from '@crawlee/core';import type { Awaitable, Dictionary } from '@crawlee/types';-import { RETRY_CSS_SELECTORS, gotScraping, type CheerioRoot } from '@crawlee/utils';+import { RETRY_CSS_SELECTORS, type CheerioRoot } from '@crawlee/utils';import * as cheerio from 'cheerio';import type { RequestLike, ResponseLike } from 'content-type';import contentTypeParser from 'content-type';import type {OptionsInit,Method,- Request as GotRequest,- Options,- PlainResponse,TimeoutError as TimeoutErrorClass,// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood} from 'got-scraping';import iconv from 'iconv-lite';import mime from 'mime-types';-import ow from 'ow';+import ow, { ObjectPredicate } from 'ow';import type { JsonValue } from 'type-fest';let TimeoutError: typeof TimeoutErrorClass;+/**+ * TODO exists for BC within HttpCrawler - replace completely with StreamingHttpResponse in 4.0+ * @internal+ */+export type PlainResponse = Omit<HttpResponse, 'body'> &+ IncomingMessage & {+ body?: unknown;+ };+/*** Default mime types, which HttpScraper supports.*/@@ -780,7 +789,7 @@ export class HttpCrawler<* Combines the provided `requestOptions` with mandatory (non-overridable) values.*/protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit) {- const requestOptions: OptionsInit & { isStream: true } = {+ const requestOptions: OptionsInit & Required<Pick<OptionsInit, 'url'>> & { isStream: true } = {url: request.url,method: request.method as Method,proxyUrl,@@ -913,34 +922,29 @@ export class HttpCrawler</*** @internal wraps public utility for mocking purposes*/- private _requestAsBrowser = async (options: OptionsInit & { isStream: true }, session?: Session) => {- // eslint-disable-next-line no-async-promise-executor- return new Promise<PlainResponse>(async (resolve, reject) => {- // This await may not be needed after the initial call, but this is needed to actually get got-scraping loaded- // eslint-disable-next-line @typescript-eslint/await-thenable- const stream = await gotScraping(options);-- stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {+ private _requestAsBrowser = async (+ options: OptionsInit & { url: string | URL; isStream: true },+ session?: Session,+ ) => {+ const response = await this.httpClient.stream(+ processHttpRequestOptions({+ ...options,+ cookieJar: options.cookieJar as any, // HACK - the type of ToughCookieJar in got is wrong+ responseType: 'text',+ }),+ (redirectResponse, updatedRequest) => {if (this.persistCookiesPerSession) {session!.setCookiesFromResponse(redirectResponse);- const cookieString = session!.getCookieString(updatedOptions.url!.toString());+ const cookieString = session!.getCookieString(updatedRequest.url!.toString());if (cookieString !== '') {- updatedOptions.headers.Cookie = cookieString;+ updatedRequest.headers.Cookie = cookieString;}}- });-- // We need to end the stream for DELETE requests, otherwise it will hang.- if (options.method && ['DELETE', 'delete'].includes(options.method)) {- stream.end();- }+ },+ );- stream.on('error', reject);- stream.on('response', () => {- resolve(addResponsePropertiesToStream(stream));- });- });+ return addResponsePropertiesToStream(response.stream, response);};}@@ -959,8 +963,8 @@ interface RequestFunctionOptions {* from the response stream to the got stream.* @internal*/-function addResponsePropertiesToStream(stream: GotRequest) {- const properties = [+function addResponsePropertiesToStream(stream: Readable, response: StreamingHttpResponse) {+ const properties: (keyof PlainResponse)[] = ['statusCode','statusMessage','headers',@@ -973,13 +977,12 @@ function addResponsePropertiesToStream(stream: GotRequest) {'request',];- const response = stream.response!;-- response.on('end', () => {+ stream.on('end', () => {// @ts-expect-error- if (stream.rawTrailers) Object.assign(stream.rawTrailers, response.rawTrailers);+ if (stream.rawTrailers) stream.rawTrailers = response.rawTrailers; // TODO BC with got - remove in 4.0+// @ts-expect-error- if (stream.trailers) Object.assign(stream.trailers, response.trailers);+ if (stream.trailers) stream.trailers = response.trailers;// @ts-expect-errorstream.complete = response.complete;@@ -987,8 +990,7 @@ function addResponsePropertiesToStream(stream: GotRequest) {for (const prop of properties) {if (!(prop in stream)) {- // @ts-expect-error- stream[prop] = response[prop as keyof PlainResponse];+ (stream as any)[prop] = (response as any)[prop];}}@@ -999,12 +1001,12 @@ function addResponsePropertiesToStream(stream: GotRequest) {* Gets parsed content type from response object* @param response HTTP response object*/-function parseContentTypeFromResponse(response: IncomingMessage): { type: string; charset: BufferEncoding } {+function parseContentTypeFromResponse(response: unknown): { type: string; charset: BufferEncoding } {ow(response,ow.object.partialShape({url: ow.string.url,- headers: ow.object,+ headers: new ObjectPredicate<Record<string, unknown>>(),}),);@@ -1013,7 +1015,7 @@ function parseContentTypeFromResponse(response: IncomingMessage): { type: stringif (headers['content-type']) {try {- parsedContentType = contentTypeParser.parse(headers['content-type']);+ parsedContentType = contentTypeParser.parse(headers['content-type'] as string);} catch {// Can not parse content type from Content-Type header. Try to parse it from file extension.}diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.tsindex 9ba820c5a..6ca3e8e7a 100644--- a/packages/utils/src/index.ts+++ b/packages/utils/src/index.ts@@ -11,5 +11,6 @@ export * from './internals/open_graph_parser';export * from './internals/gotScraping';export * from './internals/robots';export * from './internals/sitemap';+export * from './internals/url';export { Dictionary, Awaitable, Constructor } from '@crawlee/types';diff --git a/packages/utils/src/internals/url.ts b/packages/utils/src/internals/url.tsnew file mode 100644index 000000000..c13d674ea--- /dev/null+++ b/packages/utils/src/internals/url.ts@@ -0,0 +1,38 @@+export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;++/**+ * Appends search (query string) parameters to a URL, replacing the original value (if any).+ *+ * @param url The URL to append to.+ * @param searchParams The search parameters to be appended.+ * @internal+ */+export function applySearchParams(url: URL, searchParams: SearchParams | undefined): void {+ if (searchParams === undefined) {+ return;+ }++ if (typeof searchParams === 'string') {+ url.search = searchParams;+ return;+ }++ let newSearchParams: URLSearchParams;++ if (searchParams instanceof URLSearchParams) {+ newSearchParams = searchParams;+ } else {+ newSearchParams = new URLSearchParams();+ for (const [key, value] of Object.entries(newSearchParams)) {+ if (value === undefined) {+ newSearchParams.delete(key);+ } else if (value === null) {+ newSearchParams.append(key, '');+ } else {+ newSearchParams.append(key, value as string);+ }+ }+ }++ url.search = newSearchParams.toString();+}diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex 296235181..7d6d052f8 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -1,5 +1,6 @@import http from 'node:http';import type { AddressInfo } from 'node:net';+import { Readable } from 'node:stream';import { HttpCrawler } from '@crawlee/http';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';@@ -400,3 +401,57 @@ test('should work with cacheable-request', async () => {]);expect(isFromCache).toEqual({ first: false, second: true });});++test('works with a custom HttpClient', async () => {+ const results: string[] = [];++ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: async ({ body, sendRequest }) => {+ results.push(body as string);++ results.push((await sendRequest()).body);+ },+ httpClient: {+ async sendRequest(request) {+ if (request.responseType !== 'text') {+ throw new Error('Not implemented');+ }++ return {+ body: 'Hello from sendRequest()' as any,+ request,+ url,+ redirectUrls: [],+ statusCode: 200,+ headers: {},+ trailers: {},+ complete: true,+ };+ },+ async stream(request) {+ const stream = new Readable();+ stream.push('<html><head><title>Schmexample Domain</title></head></html>');+ stream.push(null);++ return {+ stream,+ downloadProgress: { percent: 100, transferred: 0 },+ uploadProgress: { percent: 100, transferred: 0 },+ request,+ url,+ redirectUrls: [],+ statusCode: 200,+ headers: { 'content-type': 'text/html; charset=utf-8' },+ trailers: {},+ complete: true,+ };+ },+ },+ });++ await crawler.run([url]);++ expect(results[0].includes('Schmexample Domain')).toBeTruthy();+ expect(results[1].includes('Hello')).toBeTruthy();+});diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/.actor/actor.json b/test/e2e/cheerio-curl-impersonate-ts/actor/.actor/actor.jsonnew file mode 100644index 000000000..323bccbdd--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/.actor/actor.json@@ -0,0 +1,7 @@+{+ "actorSpecification": 1,+ "name": "test-cheerio-curl-impersonate-ts",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/.eslintrc.json b/test/e2e/cheerio-curl-impersonate-ts/actor/.eslintrc.jsonnew file mode 100644index 000000000..26237587e--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/.eslintrc.json@@ -0,0 +1,13 @@+{+ "root": true,+ "extends": "../../.eslintrc.json",+ "parserOptions": {+ "project": "./test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json",+ "ecmaVersion": 2022+ },+ "rules": {+ "no-empty-function": "off",+ "@typescript-eslint/no-explicit-any": "off",+ "no-constant-condition": "off"+ }+}diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/.gitignore b/test/e2e/cheerio-curl-impersonate-ts/actor/.gitignorenew file mode 100644index 000000000..f2fc11c72--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/.gitignore@@ -0,0 +1,11 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storage+main.d.ts+main.d.ts.map+main.js+main.js.mapdiff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfilenew file mode 100644index 000000000..91fadb146--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/Dockerfile@@ -0,0 +1,30 @@+FROM node:20 AS builder++COPY /packages ./packages+COPY /package*.json ./+COPY /tsconfig.json ./+COPY /main.ts ./+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional --no-audit \+ && npm update \+ && npm run build++FROM lwthiker/curl-impersonate++COPY --from=builder /usr/local/bin /usr/local/bin+COPY --from=builder /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm+COPY --from=builder /node_modules ./node_modules+COPY --from=builder /packages ./packages+COPY --from=builder /package*.json ./+COPY --from=builder /main.js ./+COPY /.actor ./.actor++RUN echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++# run compiled code+CMD npm run start:proddiff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.tsnew file mode 100644index 000000000..af0d6da9e--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts@@ -0,0 +1,194 @@+import { Readable } from 'stream';++import { CheerioCrawler } from '@crawlee/cheerio';+import {+ BaseHttpClient,+ BaseHttpResponseData,+ HttpRequest,+ HttpResponse,+ RedirectHandler,+ ResponseTypes,+ StreamingHttpResponse,+} from '@crawlee/core';+import { Actor } from 'apify';+import { CurlImpersonate } from 'apify-node-curl-impersonate';++if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {+ // @ts-ignore+ await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });+} else {+ await Actor.init();+}++interface CurlImpersonateHttpClientOptions {+ impersonate?: ConstructorParameters<typeof CurlImpersonate>[1]['impersonate'];+}++type CurlResponse = Awaited<ReturnType<CurlImpersonate['makeRequest']>>;++class CurlImpersonateHttpClient implements BaseHttpClient {+ constructor(private options: CurlImpersonateHttpClientOptions = {}) {}++ protected async curlOptionsForRequest(request: HttpRequest<any>) {+ const result = {+ method: request.method ?? 'get',+ headers: request.headers ?? {},+ flags: ['--compressed'],+ impersonate: this.options.impersonate,+ debugLogger: () => {},+ };++ if (request.proxyUrl !== undefined) {+ result.flags.push('--proxy', request.proxyUrl);+ }++ if (request.cookieJar) {+ result.headers['set-cookie'] = await Promise.resolve(+ request.cookieJar.getCookieString(request.url.toString(), {}, () => {}),+ );+ }++ return result;+ }++ protected shouldRedirect(request: HttpRequest, response: CurlResponse, redirectUrls: URL[]): boolean {+ if (request.maxRedirects !== undefined && request.maxRedirects <= redirectUrls.length) {+ return false;+ }++ if (request.followRedirect instanceof Function) {+ return request.followRedirect(response);+ }++ return request.followRedirect === undefined || request.followRedirect;+ }++ protected async performRequest(request: HttpRequest<any>, onRedirect?: RedirectHandler) {+ let response: CurlResponse | undefined;+ const redirectUrls: URL[] = [];++ const updatedRequest: Required<Pick<HttpRequest, 'url' | 'headers'>> = {+ url: request.url.toString(),+ headers: {},+ };++ while (true) {+ const impersonate = new CurlImpersonate(updatedRequest.url.toString(), {+ ...(await this.curlOptionsForRequest(request)),+ followRedirects: false,+ });++ response = await impersonate.makeRequest();++ if (response.statusCode >= 300 && response.statusCode < 400) {+ if (!this.shouldRedirect(request, response, redirectUrls)) {+ break;+ }++ updatedRequest.url = response.responseHeaders.location;+ updatedRequest.headers = response.responseHeaders;+ redirectUrls.push(new URL(updatedRequest.url));+ onRedirect?.(this.transformResponse(response, redirectUrls), updatedRequest);+ } else if (request.throwHttpErrors !== false && response.statusCode >= 400) {+ throw Object.assign(new Error(`Error status code encountered - ${response.statusCode}`), {+ request,+ response,+ });+ } else {+ break;+ }+ }++ return { ...response, redirectUrls };+ }++ protected transformResponse(response: CurlResponse, redirectUrls: URL[]): BaseHttpResponseData {+ return {+ redirectUrls,+ url: response.url,+ ip: response.ipAddress,+ statusCode: response.statusCode,+ headers: response.responseHeaders,+ trailers: {}, // TODO apify-node-curl-impersonate doesn't seem to expose this+ complete: true,+ };+ }++ async sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(+ request: HttpRequest<TResponseType>,+ ): Promise<HttpResponse<TResponseType>> {+ const response = await this.performRequest(request);++ const body = ((): ResponseTypes[TResponseType] => {+ const buffer = Buffer.from(response.response, request.encoding);++ switch (request.responseType) {+ case 'buffer':+ return buffer as ResponseTypes[TResponseType];+ case 'json':+ return JSON.parse(buffer.toString());+ default:+ return buffer.toString() as ResponseTypes[TResponseType];+ }+ })();++ return {+ request,+ body,+ ...this.transformResponse(response, response.redirectUrls),+ };+ }++ async stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {+ const response = await this.performRequest(request, onRedirect);++ const stream = new Readable();+ stream.push(response.response);+ stream.push(null);++ return {+ request,+ url: response.url,+ ip: response.ipAddress,+ statusCode: response.statusCode,+ stream,+ complete: true,+ downloadProgress: { percent: 100, transferred: response.response.length },+ uploadProgress: { percent: 100, transferred: 0 },+ redirectUrls: response.redirectUrls,+ headers: response.responseHeaders,+ trailers: {},+ };+ }+}++const crawler = new CheerioCrawler({+ async requestHandler(context) {+ const { body: text } = await context.sendRequest({+ url: 'https://httpbin.org/uuid',+ });++ const { body: json } = await context.sendRequest({+ url: 'https://httpbin.org/uuid',+ responseType: 'json',+ });++ const { body: ua } = await context.sendRequest({+ url: 'https://httpbin.org/user-agent',+ responseType: 'json',+ });++ await context.pushData({+ body: context.body,+ title: context.$('title').text(),+ userAgent: ua['user-agent'],+ uuidTextResponse: text,+ uuidJsonResponse: json,+ });+ },+ httpClient: new CurlImpersonateHttpClient({ impersonate: 'chrome-116' }),+});++await crawler.run(['https://httpbin.org/']);++await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/package.json b/test/e2e/cheerio-curl-impersonate-ts/actor/package.jsonnew file mode 100644index 000000000..8e788a918--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/package.json@@ -0,0 +1,36 @@+{+ "name": "test-cheerio-curl-impersonate-ts",+ "version": "0.0.1",+ "description": "Cheerio Crawler Test - curl-impersonate HTTP client",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.3",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils",+ "apify-node-curl-impersonate": "1.0.15"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "devDependencies": {+ "@apify/tsconfig": "^0.1.0",+ "typescript": "^5.0.0"+ },+ "scripts": {+ "start": "tsc && node main.js",+ "start:prod": "node main.js",+ "build": "tsc"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json b/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.jsonnew file mode 100644index 000000000..f6f2e9d77--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/tsconfig.json@@ -0,0 +1,11 @@+{+ "extends": "@apify/tsconfig",+ "compilerOptions": {+ "module": "ES2022",+ "target": "ES2022",+ "lib": ["DOM"],+ "skipLibCheck": true,+ "incremental": false+ },+ "include": ["./**/*.ts"]+}diff --git a/test/e2e/cheerio-curl-impersonate-ts/test.mjs b/test/e2e/cheerio-curl-impersonate-ts/test.mjsnew file mode 100644index 000000000..3ff4628a3--- /dev/null+++ b/test/e2e/cheerio-curl-impersonate-ts/test.mjs@@ -0,0 +1,21 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { stats, datasetItems } = await runActor(testActorDirname);++await expect(stats.requestsFinished > 0, 'All requests finished');+await expect(datasetItems.length === 1, 'A dataset item was pushed');++const result = datasetItems[0];++expect(result.body.length > 1000, 'HTML response is not empty');+expect(result.title === 'httpbin.org', 'HTML title is correct');+expect(+ result.userAgent ===+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',+ 'User agent is chrome',+);+expect(result.uuidJsonResponse.uuid !== undefined, 'JSON response contains UUID');+expect(JSON.parse(result.uuidTextResponse).uuid !== undefined, 'Text response contains UUID');diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex b6f101a58..f82bfb836 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -281,16 +281,11 @@ async function copyPackages(dirName) {Object.assign(dependencies, overrides.apify);}- // We don't need to copy the following packages- delete dependencies['@apify/storage-local'];- delete dependencies['apify-client'];- delete dependencies['deep-equal'];- delete dependencies['playwright-core'];- delete dependencies.apify;- delete dependencies.puppeteer;- delete dependencies.playwright;-for (const dependency of Object.values(dependencies)) {+ if (!dependency.startsWith('file:')) {+ continue;+ }+const packageDirName = dependency.split('/').pop();const srcDir = join(srcPackagesDir, packageDirName, 'dist');const destDir = join(destPackagesDir, packageDirName, 'dist');diff --git a/website/sidebars.js b/website/sidebars.jsindex c7eaf4f6e..0ac1feb59 100644--- a/website/sidebars.js+++ b/website/sidebars.js@@ -45,6 +45,7 @@ module.exports = {'guides/typescript-project','guides/docker-images','guides/parallel-scraping/parallel-scraping-guide',+ 'guides/custom-http-client/custom-http-client'],},{diff --git a/yarn.lock b/yarn.lockindex 7427f8e81..6c5012392 100644--- a/yarn.lock+++ b/yarn.lock@@ -764,6 +764,7 @@ __metadata:"@typescript-eslint/parser": "npm:^7.18.0""@vitest/coverage-v8": "npm:^2.0.0"apify: "npm:*"+ apify-node-curl-impersonate: "npm:^1.0.15"basic-auth-parser: "npm:^0.0.2"body-parser: "npm:^1.20.0"commitlint: "npm:^19.0.0"@@ -3159,6 +3160,13 @@ __metadata:languageName: nodelinkType: hard+"apify-node-curl-impersonate@npm:^1.0.15":+ version: 1.0.15+ resolution: "apify-node-curl-impersonate@npm:1.0.15"+ checksum: 10c0/41a691d015bff23dc58ea1559015d7e2f40eaa3ad2e8d6cb5076358ed19effc6ce064886c66dbd6d1ed7382f3caa5c4786c0860545e2ca12ddf32fefef0d7156+ languageName: node+ linkType: hard+"apify@npm:*":version: 3.2.5resolution: "apify@npm:3.2.5"