Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: remove direct dependency on `idcac-playwright` (#3259)

Jindřich Bär6c76d32e4308
diff --git a/packages/crawlee/package.json b/packages/crawlee/package.json
index 52f57ef27..6da76a20b 100644
--- a/packages/crawlee/package.json
+++ b/packages/crawlee/package.json
@@ -70,10 +70,14 @@
"tslib": "^2.4.0"
},
"peerDependencies": {
+ "idcac-playwright": "*",
"playwright": "*",
"puppeteer": "*"
},
"peerDependenciesMeta": {
+ "idcac-playwright": {
+ "optional": true
+ },
"playwright": {
"optional": true
},
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 34c59d965..70287c15c 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -62,7 +62,6 @@
"@crawlee/types": "3.15.3",
"@crawlee/utils": "3.15.3",
"cheerio": "1.0.0-rc.12",
- "idcac-playwright": "^0.1.2",
"jquery": "^3.6.0",
"lodash.isequal": "^4.5.0",
"ml-logistic-regression": "^2.0.0",
@@ -72,9 +71,13 @@
"tslib": "^2.4.0"
},
"peerDependencies": {
+ "idcac-playwright": "^0.1.2",
"playwright": "*"
},
"peerDependenciesMeta": {
+ "idcac-playwright": {
+ "optional": true
+ },
"playwright": {
"optional": true
}
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
index ac7e1d813..ae143a833 100644
--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -33,7 +33,6 @@ import {
import type { BatchAddRequestsResult } from '@crawlee/types';
import { type CheerioRoot, type Dictionary, expandShadowRoots, sleep } from '@crawlee/utils';
import * as cheerio from 'cheerio';
-import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
import ow from 'ow';
import type { Page, Response, Route } from 'playwright';
@@ -656,10 +655,35 @@ export async function parseWithCheerio(
return cheerio.load(pageContent);
}
+let idcacPlaywright: null | { getInjectableScript: () => string } = null;
+async function getIdcacPlaywright() {
+ if (idcacPlaywright) return idcacPlaywright;
+
+ try {
+ idcacPlaywright = await import('idcac-playwright');
+ } catch (error: any) {
+ log.warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+ }
+ return idcacPlaywright;
+}
+
export async function closeCookieModals(page: Page): Promise<void> {
ow(page, ow.object.validate(validators.browserPage));
+ const idcac = await getIdcacPlaywright();
- await page.evaluate(getCookieClosingScript());
+ if (idcac?.getInjectableScript()) {
+ await page.evaluate(idcac.getInjectableScript());
+ }
}
interface HandleCloudflareChallengeOptions {
@@ -993,6 +1017,15 @@ export interface PlaywrightContextUtils {
/**
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+ *
+ * Note that this method requires the idcac-playwright package to be installed.
+ * Crawlee does not include it by default due to licensing issues.
+ *
+ * To use this method, please install the package manually by running:
+ *
+ * ```bash
+ * npm install idcac-playwright
+ * ```
*/
closeCookieModals(): Promise<void>;
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index 7c5de62f7..a84c67511 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -67,9 +67,13 @@
"tslib": "^2.4.0"
},
"peerDependencies": {
+ "idcac-playwright": "^0.1.2",
"puppeteer": "*"
},
"peerDependenciesMeta": {
+ "idcac-playwright": {
+ "optional": true
+ },
"puppeteer": {
"optional": true
}
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
index 17b097f02..8c061421f 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -27,7 +27,6 @@ import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import { type CheerioRoot, expandShadowRoots, sleep } from '@crawlee/utils';
import * as cheerio from 'cheerio';
import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
-import { getInjectableScript } from 'idcac-playwright';
import ow from 'ow';
import type { HTTPRequest as PuppeteerRequest, HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
@@ -778,8 +777,35 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}
}
}
+let idcacPlaywright: null | { getInjectableScript: () => string } = null;
+async function getIdcacPlaywright() {
+ if (idcacPlaywright) return idcacPlaywright;
+
+ try {
+ idcacPlaywright = await import('idcac-playwright');
+ } catch (error: any) {
+ log.warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+ }
+ return idcacPlaywright;
+}
+
export async function closeCookieModals(page: Page): Promise<void> {
- await page.evaluate(getInjectableScript());
+ ow(page, ow.object.validate(validators.browserPage));
+ const idcac = await getIdcacPlaywright();
+
+ if (idcac?.getInjectableScript()) {
+ await page.evaluate(idcac.getInjectableScript());
+ }
}
/** @internal */
@@ -1057,6 +1083,15 @@ export interface PuppeteerContextUtils {
/**
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+ *
+ * Note that this method requires the idcac-playwright package to be installed.
+ * Crawlee does not include it by default due to licensing issues.
+ *
+ * To use this method, please install the package manually by running:
+ *
+ * ```bash
+ * npm install idcac-playwright
+ * ```
*/
closeCookieModals(): Promise<void>;
}
diff --git a/yarn.lock b/yarn.lock
index ee86b77fa..02a0493d1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -686,7 +686,6 @@ __metadata:
"@crawlee/types": "npm:3.15.3"
"@crawlee/utils": "npm:3.15.3"
cheerio: "npm:1.0.0-rc.12"
- idcac-playwright: "npm:^0.1.2"
jquery: "npm:^3.6.0"
lodash.isequal: "npm:^4.5.0"
ml-logistic-regression: "npm:^2.0.0"
@@ -695,8 +694,11 @@ __metadata:
string-comparison: "npm:^1.3.0"
tslib: "npm:^2.4.0"
peerDependencies:
+ idcac-playwright: ^0.1.2
playwright: "*"
peerDependenciesMeta:
+ idcac-playwright:
+ optional: true
playwright:
optional: true
languageName: unknown
@@ -719,8 +721,11 @@ __metadata:
ow: "npm:^0.28.1"
tslib: "npm:^2.4.0"
peerDependencies:
+ idcac-playwright: ^0.1.2
puppeteer: "*"
peerDependenciesMeta:
+ idcac-playwright:
+ optional: true
puppeteer:
optional: true
languageName: unknown
@@ -4934,9 +4939,12 @@ __metadata:
import-local: "npm:^3.1.0"
tslib: "npm:^2.4.0"
peerDependencies:
+ idcac-playwright: "*"
playwright: "*"
puppeteer: "*"
peerDependenciesMeta:
+ idcac-playwright:
+ optional: true
playwright:
optional: true
puppeteer: