Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `ignoreIframes` opt-out from the Cheerio iframe expansion (#2562)
Jindřich Bär474a8dc06a56
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 716fcb9cb..44844981c 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -260,6 +260,12 @@ export interface BrowserCrawlerOptions<* By default, they are expanded automatically. Use this option to disable this behavior.*/ignoreShadowRoots?: boolean;++ /**+ * Whether to ignore `iframes` when processing the page content via `parseWithCheerio` helper.+ * By default, `iframes` are expanded automatically. Use this option to disable this behavior.+ */+ ignoreIframes?: boolean;}/**@@ -343,6 +349,7 @@ export abstract class BrowserCrawler<useSessionPool: ow.optional.boolean,proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),ignoreShadowRoots: ow.optional.boolean,+ ignoreIframes: ow.optional.boolean,};/**@@ -372,6 +379,7 @@ export abstract class BrowserCrawler<handleFailedRequestFunction,headless,ignoreShadowRoots,+ ignoreIframes,...basicCrawlerOptions} = options;diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 30b7ea43b..89b2a0d56 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -597,10 +597,14 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}* @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object.* @param ignoreShadowRoots*/-export async function parseWithCheerio(page: Page, ignoreShadowRoots = false): Promise<CheerioRoot> {+export async function parseWithCheerio(+ page: Page,+ ignoreShadowRoots = false,+ ignoreIframes = false,+): Promise<CheerioRoot> {ow(page, ow.object.validate(validators.browserPage));- if (page.frames().length > 1) {+ if (page.frames().length > 1 && !ignoreIframes) {const frames = await page.$$('iframe');await Promise.all(@@ -858,7 +862,7 @@ export function registerUtilsToContext(await context.waitForSelector(selector, timeoutMs);}- return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots);+ return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);};context.infiniteScroll = async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options);context.saveSnapshot = async (options?: SaveSnapshotOptions) =>diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex e3ab78b3b..ec8702538 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -188,10 +188,14 @@ export async function injectJQuery(page: Page, options?: { surviveNavigations?:* @param page Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.* @param ignoreShadowRoots*/-export async function parseWithCheerio(page: Page, ignoreShadowRoots = false): Promise<CheerioRoot> {+export async function parseWithCheerio(+ page: Page,+ ignoreShadowRoots = false,+ ignoreIframes = false,+): Promise<CheerioRoot> {ow(page, ow.object.validate(validators.browserPage));- if (page.frames().length > 1) {+ if (page.frames().length > 1 && !ignoreIframes) {const frames = await page.$$('iframe');await Promise.all(@@ -1068,7 +1072,7 @@ export function registerUtilsToContext(await context.waitForSelector(selector, timeoutMs);}- return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots);+ return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);};context.enqueueLinksByClickingElements = async (options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>,