Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(playwright): keep detecting a Cloudflare challenge that navigates itself (#4170)
Martin Adámekf90b51c9841e
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 4183c70f9..22c6b6a9b 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -703,7 +703,11 @@ export async function handleCloudflareChallenge(// check if we ended up on the CF challenge pageconst isChallenge = async () => {- return options.isChallengeCallback!(page).catch(() => false);+ return options.isChallengeCallback!(page).catch(async () => {+ // the challenge page navigates itself, which destroys the context of an in-flight check+ await page.waitForLoadState('domcontentloaded').catch(() => {});+ return options.isChallengeCallback!(page).catch(() => false);+ });};if (!(await isChallenge())) {diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex 3964d3b1e..074e38a55 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -3,6 +3,7 @@ import path from 'node:path';import { MemoryStorageBackend, serviceLocator } from '@crawlee/core';import { KeyValueStore, launchPlaywright, playwrightUtils, Request } from '@crawlee/playwright';+import { sleep } from '@crawlee/utils';import type { Browser, Page } from 'playwright';import { chromium } from 'playwright';import { runExampleComServer } from '../shared/_helper.js';@@ -495,6 +496,41 @@ describe('playwrightUtils', () => {);});+ test('detects the challenge when the challenge page navigates itself during the check', async () => {+ const url = 'https://challenge.test/';+ await page.route(`${url}**`, async (route) => {+ if (!route.request().url().includes('__cf_chl_tk')) {+ // keeps the check busy until the navigation below commits and destroys its execution context+ await route.fulfill({+ contentType: 'text/html',+ body: `<html><body>${currentChallengeBody}<script>+ window.addEventListener('load', () => setTimeout(() => { location.href = '/?__cf_chl_tk=1'; }));+ const querySelector = document.querySelector.bind(document);+ document.querySelector = (selector) => {+ const end = Date.now() + 1000;+ while (Date.now() < end) {}+ return querySelector(selector);+ };+ </script></body></html>`,+ });+ return;+ }++ await sleep(200);+ await route.fulfill({+ contentType: 'text/html',+ body: `<html><body>${currentChallengeBody}</body></html>`,+ });+ });++ const selfNavigation = page.waitForRequest((request) => request.url().includes('__cf_chl_tk'));+ await page.goto(url);+ await selfNavigation;++ await expect(handleCloudflareChallenge(page, url, fastOptions())).rejects.toThrow(/Blocked by Cloudflare/);+ await page.unrouteAll();+ });+test('resolves without detection on a regular page', async () => {await page.setContent('<html><body><h1>Welcome</h1></body></html>');await expect(