Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: prefer ImpitHttpClient browser over session fingerprint (#4262)

Jindřich Bärea99e2fe433c
diff --git a/packages/impit-client/package.json b/packages/impit-client/package.json
index 3fec26d9f..ee1ab61f4 100644
--- a/packages/impit-client/package.json
+++ b/packages/impit-client/package.json
@@ -48,6 +48,7 @@
},
"dependencies": {
"@apify/datastructures": "^3.0.1",
+ "@crawlee/core": "workspace:*",
"@crawlee/http-client": "workspace:*",
"@crawlee/types": "workspace:*",
"impit": "^0.14.2",
diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.ts
index 596ecff85..ec2cea26a 100644
--- a/packages/impit-client/src/index.ts
+++ b/packages/impit-client/src/index.ts
@@ -1,3 +1,4 @@
+import { serviceLocator } from '@crawlee/core';
import type { CustomFetchOptions } from '@crawlee/http-client';
import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
import type { CrawleeLogger, SessionFingerprint } from '@crawlee/types';
@@ -81,7 +82,14 @@ export class ImpitHttpClient extends BaseHttpClient {
options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & { cacheClients?: boolean; logger?: CrawleeLogger },
) {
super({ logger: options?.logger });
- const { cacheClients = true, logger: _logger, ...impitOptions } = options ?? {};
+ const { cacheClients = true, logger, ...impitOptions } = options ?? {};
+
+ if (impitOptions.browser) {
+ (logger ?? serviceLocator.getLogger()).warningOnce(
+ 'The `browser` option of ImpitHttpClient overrides the browser fingerprint of the session. ' +
+ 'Using it with Crawlee crawlers can lead to incompatible Session setups.',
+ );
+ }
this.#impitOptions = impitOptions;
this.#cacheClients = cacheClients;
@@ -93,7 +101,7 @@ export class ImpitHttpClient extends BaseHttpClient {
protected override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response> {
const { proxyUrl, redirect, signal, fingerprint, ignoreTlsErrors } = options ?? {};
- const impitBrowser = this.#resolveImpitBrowser(fingerprint);
+ const impitBrowser = this.#impitOptions.browser ? undefined : this.#resolveImpitBrowser(fingerprint);
const impit = this.#getClient({
...this.#impitOptions,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0b67c4c91..c65857ad6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -826,6 +826,9 @@ importers:
'@apify/datastructures':
specifier: ^3.0.1
version: 3.0.1
+ '@crawlee/core':
+ specifier: workspace:*
+ version: link:../core
'@crawlee/http-client':
specifier: workspace:*
version: link:../http-client
diff --git a/test/core/impit-http-client.test.ts b/test/core/impit-http-client.test.ts
index e6cb1db47..02c32b0f1 100644
--- a/test/core/impit-http-client.test.ts
+++ b/test/core/impit-http-client.test.ts
@@ -1,4 +1,6 @@
+import { serviceLocator } from '@crawlee/core';
import { ImpitHttpClient } from '@crawlee/impit-client';
+import type { CrawleeLogger, ISession } from '@crawlee/types';
import { Impit } from 'impit';
vi.mock('impit', () => ({
@@ -47,4 +49,40 @@ describe('ImpitHttpClient', () => {
expect(Impit).toHaveBeenCalledWith(expect.objectContaining({ ignoreTlsErrors: true }));
});
+
+ test('constructor-level browser takes precedence over the session fingerprint', async () => {
+ const httpClient = new ImpitHttpClient({ browser: 'firefox144' });
+ const session = { fingerprint: { browser: 'chrome' } } as ISession;
+
+ await httpClient.sendRequest(new Request('http://example.com'), { session });
+
+ expect(Impit).toHaveBeenCalledTimes(1);
+ expect(Impit).toHaveBeenCalledWith(expect.objectContaining({ browser: 'firefox144' }));
+ });
+
+ test('warns on the passed logger only when the browser option is set', () => {
+ const logger = { warningOnce: vi.fn() } as unknown as CrawleeLogger;
+
+ void new ImpitHttpClient({ logger });
+ expect(logger.warningOnce).not.toHaveBeenCalled();
+
+ void new ImpitHttpClient({ browser: 'firefox144', logger });
+ expect(logger.warningOnce).toHaveBeenCalledWith(expect.stringContaining('incompatible Session setups'));
+ });
+
+ test('warns on the service locator logger when no logger is passed', () => {
+ const warningOnce = vi.spyOn(serviceLocator.getLogger(), 'warningOnce').mockImplementation(() => {});
+
+ void new ImpitHttpClient({ browser: 'firefox144' });
+ expect(warningOnce).toHaveBeenCalledWith(expect.stringContaining('incompatible Session setups'));
+ });
+
+ test('uses the session fingerprint when no browser is set on the client', async () => {
+ const httpClient = new ImpitHttpClient();
+ const session = { fingerprint: { browser: 'firefox' } } as ISession;
+
+ await httpClient.sendRequest(new Request('http://example.com'), { session });
+
+ expect(Impit).toHaveBeenCalledWith(expect.objectContaining({ browser: expect.stringMatching(/^firefox/) }));
+ });
});