Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
perf: lazy-load `cheerio` in `playwright`/`puppeteer` `parseWithCheerio` (#3968)
Jindřich Bärcdab1b175c95
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 27f6ac878..ec0c9a4ae 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -25,7 +25,6 @@ import vm from 'node:vm';import { Configuration, KeyValueStore, type Request, serviceLocator, SessionError, validators } from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot, expandShadowRoots, sleep } from '@crawlee/utils';-import * as cheerio from 'cheerio';import ow from 'ow';import type { Download, Page, Response, Route } from 'playwright';@@ -611,7 +610,8 @@ export async function parseWithCheerio(? null: ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);const pageContent = html || (await page.content());- const $ = cheerio.load(pageContent);+ const { load } = await import('cheerio');+ const $ = load(pageContent);if (page.frames().length > 1 && !ignoreIframes) {const frames = await page.$$('iframe');diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex efc92f3d0..3de59ad7a 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -26,7 +26,6 @@ import type { Request } from '@crawlee/browser';import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type CheerioRoot, expandShadowRoots, sleep } from '@crawlee/utils';-import * as cheerio from 'cheerio';import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';import ow from 'ow';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll gooooood@@ -235,7 +234,8 @@ export async function parseWithCheerio(: ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);const pageContent = html || (await page.content());- return cheerio.load(pageContent);+ const { load } = await import('cheerio');+ return load(pageContent);}/**