Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(playwright): update handleCloudflareChallenge for new Cloudflare challenge markup (#4019)
Martin Adámek379f629b657c
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex c7295524b..5fdb3bb89 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -755,9 +755,9 @@ async function handleCloudflareChallenge(options.isChallengeCallback ??= async () => {return await page.evaluate(async () => {- // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants- // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).- return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');+ // Cloudflare keeps reshuffling the wrapper elements between `.footer-inner` and `.ray-id`,+ // so only the stable outer classes are matched.+ return !!document.querySelector('.footer .footer-inner .ray-id');});};diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex b765e5462..83ab2a8a5 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -433,4 +433,70 @@ describe('playwrightUtils', () => {await browser.close();}});++ describe('handleCloudflareChallenge() challenge detection', () => {+ // not a named export, only reachable via the internal `playwrightUtils` object+ const { handleCloudflareChallenge } = playwrightUtils.playwrightUtils;+ let browser: Browser;+ let page: Page;++ // no clicking or waiting, so an unsolved challenge fails fast instead of spending ~20s in the click loop+ const fastOptions = () => ({+ sleepSecs: 0,+ preChallengeSleepSecs: 0,+ clickCallback: async () => {},+ });++ // markup as rendered by the current (2026) Cloudflare challenge template+ const currentChallengeBody = `+ <div class="main-wrapper" role="main"><div class="main-content">+ <h1>example.com</h1>+ <h2>Performing security verification</h2>+ <div style="display: grid;"><div><div><input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-2swmi_response"></div></div></div>+ </div></div>+ <div class="footer" role="contentinfo"><div class="footer-inner"><div class="footer-wrapper">+ <div><div class="ray-id">Ray ID: <code>a29d9a9cff4fb9e4</code></div></div>+ <div class="footer-link-wrapper"><span class="footer-text">Performance and Security by Cloudflare</span></div>+ </div></div></div>`;++ // the older template, with the `.diagnostic-wrapper` element+ const oldChallengeBody = `+ <div class="main-wrapper" role="main"><div class="main-content">+ <h1>example.com</h1>+ <div><input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-2swmi_response"></div>+ </div></div>+ <div class="footer" role="contentinfo"><div class="footer-inner"><div class="footer-wrapper"><div class="diagnostic-wrapper">+ <div class="ray-id">Ray ID: <code>a29d9a9cff4fb9e4</code></div>+ </div></div></div></div>`;++ beforeAll(async () => {+ browser = await chromium.launch({ headless: true });+ page = await browser.newPage();+ });++ afterAll(async () => {+ await browser.close();+ });++ test('detects the current challenge markup', async () => {+ await page.setContent(`<html><body>${currentChallengeBody}</body></html>`);+ await expect(+ handleCloudflareChallenge(page, 'https://example.com', undefined, fastOptions()),+ ).rejects.toThrow(/Blocked by Cloudflare/);+ });++ test('detects the old challenge markup', async () => {+ await page.setContent(`<html><body>${oldChallengeBody}</body></html>`);+ await expect(+ handleCloudflareChallenge(page, 'https://example.com', undefined, fastOptions()),+ ).rejects.toThrow(/Blocked by Cloudflare/);+ });++ test('resolves without detection on a regular page', async () => {+ await page.setContent('<html><body><h1>Welcome</h1></body></html>');+ await expect(+ handleCloudflareChallenge(page, 'https://example.com', undefined, fastOptions()),+ ).resolves.toBeUndefined();+ });+ });});diff --git a/test/e2e/camoufox-cloudflare/actor/main.js b/test/e2e/camoufox-cloudflare/actor/main.jsindex 4a55d2e8c..3667ebd6a 100644--- a/test/e2e/camoufox-cloudflare/actor/main.js+++ b/test/e2e/camoufox-cloudflare/actor/main.js@@ -52,7 +52,7 @@ await Actor.main(async () => {async requestHandler({ page, parseWithCheerio }) {const isBlocked = await page.evaluate(async () => {- return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');+ return !!document.querySelector('.footer .footer-inner .ray-id');}).catch(() => false);const $ = await parseWithCheerio();