Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: `impit`-based `HttpClient` implementation (#2787)

Jindřich Bär61d7ffae8aff
diff --git a/packages/impit-client/README.md b/packages/impit-client/README.md
new file mode 100644
index 000000000..e9a96fba9
--- /dev/null
+++ b/packages/impit-client/README.md
@@ -0,0 +1,32 @@
+# @crawlee/impit-client
+
+This package provides a Crawlee-compliant `HttpClient` interface for the [`impit`](https://www.npmjs.com/package/impit) package.
+
+To use the `impit` package directly without Crawlee, check out [`impit`](https://www.npmjs.com/package/impit) on NPM.
+
+## Example usage
+
+Simply pass the `ImpitHttpClient` instance to the `httpClient` option of the crawler constructor:
+
+```typescript
+import { CheerioCrawler, Dictionary } from '@crawlee/cheerio';
+import { ImpitHttpClient, Browser } from '@crawlee/impit-client';
+
+const crawler = new CheerioCrawler({
+ httpClient: new ImpitHttpClient({
+ browser: Browser.Firefox,
+ http3: true,
+ ignoreTlsErrors: true,
+ }),
+ async requestHandler({ $, request }) {
+ // Extract the title of the page.
+ const title = $('title').text();
+ console.log(`Title of the page ${request.url}: ${title}`);
+ },
+});
+
+crawler.run([
+ 'http://www.example.com/page-1',
+ 'http://www.example.com/page-2',
+]);
+```
diff --git a/packages/impit-client/package.json b/packages/impit-client/package.json
new file mode 100644
index 000000000..fe60931ea
--- /dev/null
+++ b/packages/impit-client/package.json
@@ -0,0 +1,65 @@
+{
+ "name": "@crawlee/impit-client",
+ "version": "3.12.1",
+ "description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
+ "engines": {
+ "node": ">=16.0.0"
+ },
+ "main": "./dist/index.js",
+ "module": "./dist/index.mjs",
+ "types": "./dist/index.d.ts",
+ "exports": {
+ ".": {
+ "import": "./dist/index.mjs",
+ "require": "./dist/index.js",
+ "types": "./dist/index.d.ts"
+ },
+ "./package.json": "./package.json"
+ },
+ "keywords": [
+ "apify",
+ "headless",
+ "chrome",
+ "puppeteer",
+ "crawler",
+ "scraper"
+ ],
+ "author": {
+ "name": "Apify",
+ "email": "[email protected]",
+ "url": "https://apify.com"
+ },
+ "contributors": [
+ "Jan Curn <[email protected]>",
+ "Marek Trunkat <[email protected]>",
+ "Ondra Urban <[email protected]>"
+ ],
+ "license": "Apache-2.0",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/apify/crawlee"
+ },
+ "bugs": {
+ "url": "https://github.com/apify/crawlee/issues"
+ },
+ "homepage": "https://crawlee.dev",
+ "scripts": {
+ "build": "yarn clean && yarn compile && yarn copy",
+ "clean": "rimraf ./dist",
+ "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
+ "copy": "tsx ../../scripts/copy.ts"
+ },
+ "publishConfig": {
+ "access": "public"
+ },
+ "peerDependencies": {
+ "@crawlee/core": "^3.12.1"
+ },
+ "devDependencies": {
+ "@crawlee/core": "^3.12.1"
+ },
+ "dependencies": {
+ "impit": "^0.1.3"
+ },
+ "packageManager": "[email protected]"
+}
diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.ts
new file mode 100644
index 000000000..de854ea81
--- /dev/null
+++ b/packages/impit-client/src/index.ts
@@ -0,0 +1,188 @@
+import { Readable } from 'stream';
+import { isTypedArray } from 'util/types';
+
+import type { HttpRequest, HttpResponse, ResponseTypes, StreamingHttpResponse, BaseHttpClient } from '@crawlee/core';
+import { type ImpitOptions, type HttpMethod, Impit } from 'impit';
+
+export { Browser } from 'impit';
+
+/**
+ * A HTTP client implementation based on the `impit library.
+ */
+export class ImpitHttpClient implements BaseHttpClient {
+ private impitOptions: ImpitOptions;
+ private maxRedirects: number;
+ private followRedirects: boolean;
+
+ constructor(options?: Omit<ImpitOptions, 'proxyUrl'> & { maxRedirects?: number }) {
+ this.impitOptions = options ?? {};
+
+ this.maxRedirects = options?.maxRedirects ?? 10;
+ this.followRedirects = options?.followRedirects ?? true;
+ }
+
+ /**
+ * Converts the body of a `HttpRequest` to a format that can be passed to `impit`.
+ */
+ private async intoImpitBody<TResponseType extends keyof ResponseTypes>(
+ body: Exclude<HttpRequest<TResponseType>['body'], undefined>,
+ ): Promise<string | Uint8Array> {
+ if (typeof body === 'string' || isTypedArray(body)) {
+ return body;
+ }
+
+ if (body instanceof ReadableStream) {
+ const reader = body.getReader();
+ const buffer = new Uint8Array();
+
+ while (true) {
+ const { done, value } = await reader.read();
+
+ if (done) return buffer;
+
+ buffer.set(value, buffer.length);
+ }
+ }
+
+ throw new Error('Unsupported body type.');
+ }
+
+ /**
+ * Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`.
+ * @param headers `SimpleHeaders` object
+ * @returns `Record<string, string>` object
+ */
+ private flattenHeaders<TResponseType extends keyof ResponseTypes>(
+ headers: Exclude<HttpRequest<TResponseType>['headers'], undefined>,
+ ): Record<string, string> {
+ const result: Record<string, string> = {};
+
+ for (const headerName of Object.keys(headers)) {
+ const headerValue = headers[headerName];
+
+ if (headerValue === undefined) continue;
+
+ if (Array.isArray(headerValue)) {
+ result[headerName] = headerValue[0];
+ continue;
+ }
+
+ result[headerName] = headerValue;
+ }
+
+ return result;
+ }
+
+ /**
+ * Common implementation for `sendRequest` and `stream` methods.
+ * @param request `HttpRequest` object
+ * @returns `HttpResponse` object
+ */
+ private async performRequest<TResponseType extends keyof ResponseTypes>(
+ request: HttpRequest<TResponseType>,
+ redirects?: {
+ redirectCount?: number;
+ redirectUrls?: URL[];
+ },
+ ): Promise<HttpResponse<TResponseType>> {
+ if ((redirects?.redirectCount ?? 0) > this.maxRedirects) {
+ throw new Error(`Too many redirects, maximum is ${this.maxRedirects}.`);
+ }
+
+ const url = typeof request.url === 'string' ? request.url : request.url.href;
+ const headers = request.headers !== undefined ? this.flattenHeaders(request.headers) : undefined;
+ const body = request.body !== undefined ? await this.intoImpitBody(request.body) : undefined;
+
+ const impit = new Impit({
+ ...this.impitOptions,
+ proxyUrl: request.proxyUrl,
+ followRedirects: false,
+ });
+
+ const response = await impit.fetch(url, {
+ method: request.method as HttpMethod,
+ headers,
+ body: body as string,
+ });
+
+ if (this.followRedirects && response.status >= 300 && response.status < 400) {
+ const location = response.headers.location;
+
+ if (!location) {
+ throw new Error('Redirect response missing location header.');
+ }
+
+ return this.performRequest(
+ {
+ ...request,
+ url: location,
+ },
+ {
+ redirectCount: (redirects?.redirectCount ?? 0) + 1,
+ redirectUrls: [...(redirects?.redirectUrls ?? []), new URL(location)],
+ },
+ );
+ }
+
+ let responseBody;
+
+ switch (request.responseType) {
+ case 'text':
+ responseBody = response.text();
+ break;
+ case 'json':
+ responseBody = response.json();
+ break;
+ case 'buffer':
+ responseBody = response.bytes();
+ break;
+ default:
+ throw new Error('Unsupported response type.');
+ }
+
+ return {
+ headers: response.headers,
+ statusCode: response.status,
+ url,
+ request,
+ redirectUrls: redirects?.redirectUrls ?? [],
+ trailers: {},
+ body: responseBody,
+ complete: true,
+ };
+ }
+
+ /**
+ * @inheritDoc
+ */
+ async sendRequest<TResponseType extends keyof ResponseTypes>(
+ request: HttpRequest<TResponseType>,
+ ): Promise<HttpResponse<TResponseType>> {
+ return this.performRequest(request);
+ }
+
+ /**
+ * @inheritDoc
+ */
+ async stream(request: HttpRequest): Promise<StreamingHttpResponse> {
+ const response = await this.performRequest(request);
+
+ const stream = new Readable();
+ stream.push(response.body);
+ stream.push(null);
+
+ return {
+ request,
+ url: response.url,
+ ip: response.ipAddress,
+ statusCode: response.statusCode,
+ stream,
+ complete: true,
+ downloadProgress: { percent: 100, transferred: response.body.length },
+ uploadProgress: { percent: 100, transferred: 0 },
+ redirectUrls: response.redirectUrls,
+ headers: response.headers,
+ trailers: {},
+ };
+ }
+}
diff --git a/packages/impit-client/tsconfig.build.json b/packages/impit-client/tsconfig.build.json
new file mode 100644
index 000000000..2e04e9e9f
--- /dev/null
+++ b/packages/impit-client/tsconfig.build.json
@@ -0,0 +1,7 @@
+{
+ "extends": "../../tsconfig.build.json",
+ "compilerOptions": {
+ "outDir": "./dist"
+ },
+ "include": ["src/**/*"]
+}
diff --git a/packages/impit-client/tsconfig.json b/packages/impit-client/tsconfig.json
new file mode 100644
index 000000000..2e6a4ce40
--- /dev/null
+++ b/packages/impit-client/tsconfig.json
@@ -0,0 +1,4 @@
+{
+ "extends": "../../tsconfig.json",
+ "include": ["src/**/*"]
+}
diff --git a/test/e2e/cheerio-impit-ts/actor/.actor/actor.json b/test/e2e/cheerio-impit-ts/actor/.actor/actor.json
new file mode 100644
index 000000000..99324939c
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/.actor/actor.json
@@ -0,0 +1,7 @@
+{
+ "actorSpecification": 1,
+ "name": "test-cheerio-impit-ts",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/cheerio-impit-ts/actor/.eslintrc.json b/test/e2e/cheerio-impit-ts/actor/.eslintrc.json
new file mode 100644
index 000000000..629299be5
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/.eslintrc.json
@@ -0,0 +1,13 @@
+{
+ "root": true,
+ "extends": "../../.eslintrc.json",
+ "parserOptions": {
+ "project": "./test/e2e/cheerio-impit-ts/actor/tsconfig.json",
+ "ecmaVersion": 2022
+ },
+ "rules": {
+ "no-empty-function": "off",
+ "@typescript-eslint/no-explicit-any": "off",
+ "no-constant-condition": "off"
+ }
+}
diff --git a/test/e2e/cheerio-impit-ts/actor/.gitignore b/test/e2e/cheerio-impit-ts/actor/.gitignore
new file mode 100644
index 000000000..f2fc11c72
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/.gitignore
@@ -0,0 +1,11 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
+main.d.ts
+main.d.ts.map
+main.js
+main.js.map
diff --git a/test/e2e/cheerio-impit-ts/actor/Dockerfile b/test/e2e/cheerio-impit-ts/actor/Dockerfile
new file mode 100644
index 000000000..ed192b5e1
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/Dockerfile
@@ -0,0 +1,28 @@
+# using multistage build, as we need dev deps to build the TS source code
+FROM apify/actor-node:20-beta AS builder
+
+# copy all files, install all dependencies (including dev deps) and build the project
+COPY . ./
+RUN npm install --include=dev \
+ && npm run build
+
+# create final image
+FROM apify/actor-node:20-beta
+# copy only necessary files
+COPY --from=builder /usr/src/app/packages ./packages
+COPY --from=builder /usr/src/app/package.json ./
+COPY --from=builder /usr/src/app/main.js ./
+
+# install only prod deps
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-audit \
+ && npm update --no-audit \
+ && echo "Installed NPM packages:" \
+ && (npm list --omit=dev --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+# run compiled code
+CMD npm run start:prod
diff --git a/test/e2e/cheerio-impit-ts/actor/main.ts b/test/e2e/cheerio-impit-ts/actor/main.ts
new file mode 100644
index 000000000..64f2aeb77
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/main.ts
@@ -0,0 +1,41 @@
+import { CheerioCrawler, Dictionary } from '@crawlee/cheerio';
+import { ImpitHttpClient, Browser } from '@crawlee/impit-client';
+import { Actor } from 'apify';
+
+if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {
+ // @ts-ignore
+ await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });
+} else {
+ await Actor.init();
+}
+
+const crawler = new CheerioCrawler({
+ async requestHandler(context) {
+ const { body: text } = await context.sendRequest({
+ url: 'https://httpbin.org/uuid',
+ });
+
+ const { body: json } = await context.sendRequest({
+ url: 'https://httpbin.org/uuid',
+ responseType: 'json',
+ });
+
+ const { body: ua } = await context.sendRequest<Dictionary>({
+ url: 'https://httpbin.org/user-agent',
+ responseType: 'json',
+ });
+
+ await context.pushData({
+ body: context.body,
+ title: context.$('title').text(),
+ userAgent: ua['user-agent'],
+ uuidTextResponse: text,
+ uuidJsonResponse: json,
+ });
+ },
+ httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),
+});
+
+await crawler.run(['https://httpbin.org/']);
+
+await Actor.exit({ exit: Actor.isAtHome() });
diff --git a/test/e2e/cheerio-impit-ts/actor/package.json b/test/e2e/cheerio-impit-ts/actor/package.json
new file mode 100644
index 000000000..03ccac5e7
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/package.json
@@ -0,0 +1,36 @@
+{
+ "name": "test-cheerio-impit-ts",
+ "version": "0.0.1",
+ "description": "Cheerio Crawler Test - Impit HTTP client",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.3",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/browser-pool": "file:./packages/browser-pool",
+ "@crawlee/http": "file:./packages/http-crawler",
+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils",
+ "@crawlee/impit-client": "file:./packages/impit-client"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "devDependencies": {
+ "@apify/tsconfig": "^0.1.0",
+ "typescript": "^5.0.0"
+ },
+ "scripts": {
+ "start": "tsc && node main.js",
+ "start:prod": "node main.js",
+ "build": "tsc"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/cheerio-impit-ts/actor/tsconfig.json b/test/e2e/cheerio-impit-ts/actor/tsconfig.json
new file mode 100644
index 000000000..f6f2e9d77
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/actor/tsconfig.json
@@ -0,0 +1,11 @@
+{
+ "extends": "@apify/tsconfig",
+ "compilerOptions": {
+ "module": "ES2022",
+ "target": "ES2022",
+ "lib": ["DOM"],
+ "skipLibCheck": true,
+ "incremental": false
+ },
+ "include": ["./**/*.ts"]
+}
diff --git a/test/e2e/cheerio-impit-ts/test.mjs b/test/e2e/cheerio-impit-ts/test.mjs
new file mode 100644
index 000000000..3ee48b537
--- /dev/null
+++ b/test/e2e/cheerio-impit-ts/test.mjs
@@ -0,0 +1,17 @@
+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { stats, datasetItems } = await runActor(testActorDirname);
+
+await expect(stats.requestsFinished > 0, 'All requests finished');
+await expect(datasetItems.length === 1, 'A dataset item was pushed');
+
+const result = datasetItems[0];
+
+expect(result.body.length > 1000, 'HTML response is not empty');
+expect(result.title === 'httpbin.org', 'HTML title is correct');
+expect(/Gecko\/\d{8} Firefox\/\d{2}/.test(result.userAgent), 'Impit correctly spoofs Firefox');
+expect(result.uuidJsonResponse.uuid !== undefined, 'JSON response contains UUID');
+expect(JSON.parse(result.uuidTextResponse).uuid !== undefined, 'Text response contains UUID');
diff --git a/yarn.lock b/yarn.lock
index c1397e150..0cd5365a3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -579,7 +579,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@crawlee/core@npm:3.12.1, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":
+"@crawlee/core@npm:3.12.1, @crawlee/core@npm:^3.12.1, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":
version: 0.0.0-use.local
resolution: "@crawlee/core@workspace:packages/core"
dependencies:
@@ -629,6 +629,17 @@ __metadata:
languageName: unknown
linkType: soft
+"@crawlee/impit-client@workspace:packages/impit-client":
+ version: 0.0.0-use.local
+ resolution: "@crawlee/impit-client@workspace:packages/impit-client"
+ dependencies:
+ "@crawlee/core": "npm:^3.12.1"
+ impit: "npm:^0.1.3"
+ peerDependencies:
+ "@crawlee/core": ^3.12.1
+ languageName: unknown
+ linkType: soft
+
"@crawlee/jsdom@npm:3.12.1, @crawlee/jsdom@workspace:packages/jsdom-crawler":
version: 0.0.0-use.local
resolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"
@@ -7234,6 +7245,75 @@ __metadata:
languageName: node
linkType: hard
+"impit-darwin-arm64@npm:0.1.3":
+ version: 0.1.3
+ resolution: "impit-darwin-arm64@npm:0.1.3"
+ conditions: os=darwin & cpu=arm64
+ languageName: node
+ linkType: hard
+
+"impit-darwin-x64@npm:0.1.3":
+ version: 0.1.3
+ resolution: "impit-darwin-x64@npm:0.1.3"
+ conditions: os=darwin & cpu=x64
+ languageName: node
+ linkType: hard
+
+"impit-linux-x64-gnu@npm:0.1.3":
+ version: 0.1.3
+ resolution: "impit-linux-x64-gnu@npm:0.1.3"
+ conditions: os=linux & cpu=x64 & libc=glibc
+ languageName: node
+ linkType: hard
+
+"impit-linux-x64-musl@npm:0.1.3":
+ version: 0.1.3
+ resolution: "impit-linux-x64-musl@npm:0.1.3"
+ conditions: os=linux & cpu=x64 & libc=musl
+ languageName: node
+ linkType: hard
+
+"impit-win32-arm64-msvc@npm:0.1.3":
+ version: 0.1.3
+ resolution: "impit-win32-arm64-msvc@npm:0.1.3"
+ conditions: os=win32 & cpu=arm64
+ languageName: node
+ linkType: hard
+
+"impit-win32-x64-msvc@npm:0.1.3":
+ version: 0.1.3
+ resolution: "impit-win32-x64-msvc@npm:0.1.3"
+ conditions: os=win32 & cpu=x64
+ languageName: node
+ linkType: hard
+
+"impit@npm:^0.1.3":
+ version: 0.1.3
+ resolution: "impit@npm:0.1.3"
+ dependencies:
+ impit-darwin-arm64: "npm:0.1.3"
+ impit-darwin-x64: "npm:0.1.3"
+ impit-linux-x64-gnu: "npm:0.1.3"
+ impit-linux-x64-musl: "npm:0.1.3"
+ impit-win32-arm64-msvc: "npm:0.1.3"
+ impit-win32-x64-msvc: "npm:0.1.3"
+ dependenciesMeta:
+ impit-darwin-arm64:
+ optional: true
+ impit-darwin-x64:
+ optional: true
+ impit-linux-x64-gnu:
+ optional: true
+ impit-linux-x64-musl:
+ optional: true
+ impit-win32-arm64-msvc:
+ optional: true
+ impit-win32-x64-msvc:
+ optional: true
+ checksum: 10c0/0339d1c5fe11399e4a5a8bc4f5a1b1308ca3428ccc01d8e2b9eae49aa17a1163bc6e9fcdea26c6caf142fb6e0a00aa24a59e9d996224710b5112d3da4a0658c3
+ languageName: node
+ linkType: hard
+
"import-fresh@npm:^3.2.1, import-fresh@npm:^3.3.0":
version: 3.3.0
resolution: "import-fresh@npm:3.3.0"