Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: extract only `body` from `iframe` elements (#2986)
Jindřich Bärc36166e24887
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 4531f43cb..ac7e1d813 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -623,7 +623,15 @@ export async function parseWithCheerio(const iframe = await frame.contentFrame();if (iframe) {- const contents = await iframe.content();+ const getIframeHTML = async (): Promise<string> => {+ try {+ return iframe.locator('body').first().innerHTML();+ } catch {+ return iframe.content();+ }+ };++ const contents = await getIframeHTML();await frame.evaluate((f, c) => {const replacementNode = document.createElement('div');diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex 425415ce3..17b097f02 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -204,7 +204,15 @@ export async function parseWithCheerio(try {const iframe = await frame.contentFrame();if (iframe) {- const contents = await iframe.content();+ const getIframeHTML = async (): Promise<string> => {+ try {+ return iframe.$eval('body', (el) => el.innerHTML);+ } catch {+ return iframe.content();+ }+ };++ const contents = await getIframeHTML();await frame.evaluate((f, c) => {const replacementNode = document.createElement('div');diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex 8d861f0b7..dc038b5c2 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -170,8 +170,14 @@ describe('playwrightUtils', () => {const $ = await playwrightUtils.parseWithCheerio(page);const headings = $('h1')- .map((i, el) => $(el).text())+ .map((_, el) => $(el).text()).get();++ const titles = $('title')+ .map((_, el) => $(el).text())+ .get();++ expect(titles).toEqual(['Outside iframe title']);expect(headings).toEqual(['Outside iframe', 'In iframe']);} finally {await browser.close();diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.tsindex 29b0902e9..50c157827 100644--- a/test/core/puppeteer_utils.test.ts+++ b/test/core/puppeteer_utils.test.ts@@ -171,8 +171,14 @@ describe('puppeteerUtils', () => {const $ = await puppeteerUtils.parseWithCheerio(page);const headings = $('h1')- .map((i, el) => $(el).text())+ .map((_, el) => $(el).text()).get();++ const titles = $('title')+ .map((_, el) => $(el).text())+ .get();++ expect(titles).toEqual(['Outside iframe title']);expect(headings).toEqual(['Outside iframe', 'In iframe']);} finally {await browser.close();diff --git a/test/shared/_helper.ts b/test/shared/_helper.tsindex eb9822a14..8c275ea9d 100644--- a/test/shared/_helper.ts+++ b/test/shared/_helper.ts@@ -176,7 +176,7 @@ console.log('Hello world!');<!DOCTYPE html><html><head>- <title>Outside iframe</title>+ <title>Outside iframe title</title></head><body><h1>Outside iframe</h1>