Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(impit-client): add `cacheClients` option to disable `Impit` client reuse (#3771)
Copilotaf9cb8c68c82
diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.tsindex ee1e840e2..ee9407319 100644--- a/packages/impit-client/src/index.ts+++ b/packages/impit-client/src/index.ts@@ -26,6 +26,7 @@ export class ImpitHttpClient implements BaseHttpClient {private impitOptions: ImpitOptions;private maxRedirects: number;private followRedirects: boolean;+ private cacheClients: boolean;/*** Enables reuse of `impit` clients for the same set of options.@@ -35,7 +36,11 @@ export class ImpitHttpClient implements BaseHttpClient {*/private clientCache: LruCache<{ client: Impit; cookieJar: ToughCookieJar }> = new LruCache({ maxLength: 10 });- private getClient(options: ImpitOptions) {+ private getClient(options: ImpitOptions): Impit {+ if (!this.cacheClients) {+ return new Impit(options);+ }+const { cookieJar, ...rest } = options;const cacheKey = JSON.stringify(rest);@@ -51,11 +56,16 @@ export class ImpitHttpClient implements BaseHttpClient {return client;}- constructor(options?: Omit<ImpitOptions, 'proxyUrl'> & { maxRedirects?: number }) {- this.impitOptions = options ?? {};+ /**+ * @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.+ */+ constructor(options?: Omit<ImpitOptions, 'proxyUrl'> & { maxRedirects?: number; cacheClients?: boolean }) {+ const { maxRedirects = 10, followRedirects = true, cacheClients = true, ...impitOptions } = options ?? {};- this.maxRedirects = options?.maxRedirects ?? 10;- this.followRedirects = options?.followRedirects ?? true;+ this.impitOptions = impitOptions;+ this.maxRedirects = maxRedirects;+ this.followRedirects = followRedirects;+ this.cacheClients = cacheClients;}/**diff --git a/test/core/impit_http_client.test.ts b/test/core/impit_http_client.test.tsnew file mode 100644index 000000000..d85c8b427--- /dev/null+++ b/test/core/impit_http_client.test.ts@@ -0,0 +1,34 @@+import { ImpitHttpClient } from '@crawlee/impit-client';+import { Impit } from 'impit';++vi.mock('impit', () => ({+ Impit: vi.fn(+ class {+ fetch = vi.fn();+ },+ ),+}));++describe('ImpitHttpClient', () => {+ beforeEach(() => {+ vi.mocked(Impit).mockClear();+ });++ test('reuses cached clients by default', () => {+ const httpClient = new ImpitHttpClient();++ (httpClient as any).getClient({ proxyUrl: 'http://proxy.example' });+ (httpClient as any).getClient({ proxyUrl: 'http://proxy.example' });++ expect(Impit).toHaveBeenCalledTimes(1);+ });++ test('creates a new client for each request when cacheClients is false', () => {+ const httpClient = new ImpitHttpClient({ cacheClients: false });++ (httpClient as any).getClient({ proxyUrl: 'http://proxy.example' });+ (httpClient as any).getClient({ proxyUrl: 'http://proxy.example' });++ expect(Impit).toHaveBeenCalledTimes(2);+ });+});