Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: use `getHTML` in the shadow root expansion (#2587)

Jindřich Bära244d62cca03
diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.ts
index 221c84d4b..af748a333 100644
--- a/packages/utils/src/internals/general.ts
+++ b/packages/utils/src/internals/general.ts
@@ -115,7 +115,12 @@ export function expandShadowRoots(document: Document): string {
for (const el of rootElement.querySelectorAll('*')) {
if (el.shadowRoot) {
replaceShadowDomsWithHtml(el.shadowRoot);
- el.innerHTML += getShadowDomHtml(el.shadowRoot) ?? '';
+ let content = el.getHTML?.({ serializableShadowRoots: true }).trim();
+
+ if (!(content?.length > 0)) {
+ content = getShadowDomHtml(el.shadowRoot) ?? '';
+ }
+ el.innerHTML += content;
}
}
}
diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
index 67d6b0942..936718a68 100644
--- a/test/core/playwright_utils.test.ts
+++ b/test/core/playwright_utils.test.ts
@@ -244,6 +244,36 @@ describe('playwrightUtils', () => {
}
}, 60_000);
+ describe('shadow root expansion', () => {
+ let browser: Browser;
+ beforeAll(async () => {
+ browser = await launchPlaywright(launchContext);
+ });
+ afterAll(async () => {
+ await browser.close();
+ });
+
+ test('no expansion with ignoreShadowRoots: true', async () => {
+ const page = await browser.newPage();
+ await page.goto(`${serverAddress}/special/shadow-root`);
+ const result = await playwrightUtils.parseWithCheerio(page, true);
+
+ const text = result('body').text().trim();
+ expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(0);
+ expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0);
+ });
+
+ test('expansion works', async () => {
+ const page = await browser.newPage();
+ await page.goto(`${serverAddress}/special/shadow-root`);
+ const result = await playwrightUtils.parseWithCheerio(page);
+
+ const text = result('body').text().trim();
+ expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(2);
+ expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0);
+ });
+ });
+
describe('infiniteScroll()', () => {
function isAtBottom() {
return window.innerHeight + window.pageYOffset >= document.body.offsetHeight;
diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts
index 13be599f1..e46a7239d 100644
--- a/test/core/puppeteer_utils.test.ts
+++ b/test/core/puppeteer_utils.test.ts
@@ -178,6 +178,36 @@ describe('puppeteerUtils', () => {
}
});
+ describe('parseWithCheerio() shadow root expansion works', () => {
+ let browser: Browser;
+ beforeAll(async () => {
+ browser = await launchPuppeteer(launchContext);
+ });
+ afterAll(async () => {
+ await browser.close();
+ });
+
+ test('no expansion with ignoreShadowRoots: true', async () => {
+ const page = await browser.newPage();
+ await page.goto(`${serverAddress}/special/shadow-root`);
+ const result = await puppeteerUtils.parseWithCheerio(page, true);
+
+ const text = result('body').text().trim();
+ expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(0);
+ expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0);
+ });
+
+ test('expansion works', async () => {
+ const page = await browser.newPage();
+ await page.goto(`${serverAddress}/special/shadow-root`);
+ const result = await puppeteerUtils.parseWithCheerio(page);
+
+ const text = result('body').text().trim();
+ expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(2);
+ expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0);
+ });
+ });
+
describe('blockRequests()', () => {
let browser: Browser = null;
beforeAll(async () => {
diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
index a797cb256..c309465b1 100644
--- a/test/shared/_helper.ts
+++ b/test/shared/_helper.ts
@@ -194,6 +194,31 @@ console.log('Hello world!');
<p>Some content from inside of an iframe.</p>
</body>
</html>`,
+ shadowRoots: `
+ <html>
+ <body>
+ <div id="open-container">
+ <template shadowrootmode="open">
+ <p>[GOOD] This is inside the OPEN shadow DOM.</p>
+ <div>
+ <template shadowrootmode="open">
+ <p>[GOOD] This is inside of inside of the OPEN shadow DOM.</p>
+ </template>
+ </div>
+ </template>
+ </div>
+ <div id="closed-container">
+ <template shadowrootmode="closed">
+ <p>[BAD] This is inside the CLOSED shadow DOM.</p>
+ <div>
+ <template shadowrootmode="open">
+ <p>[BAD] This is inside of inside of the CLOSED shadow DOM.</p>
+ </template>
+ </div>
+ </template>
+ </div>
+ </body>
+ </html>`,
};
export async function runExampleComServer(): Promise<[Server, number]> {
@@ -298,6 +323,10 @@ export async function runExampleComServer(): Promise<[Server, number]> {
special.get('/inside-iframe', (_req, res) => {
res.type('html').send(responseSamples.insideIframe);
});
+
+ special.get('/shadow-root', (_req, res) => {
+ res.type('html').send(responseSamples.shadowRoots);
+ });
})();
// "cacheable" site with one page, scripts and stylesheets