Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: `retryOnBlocked` doesn't override the blocked HTTP codes (#2243)
Jindřich Bär81672c3d1db1
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 968460ee5..af7790630 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -25,6 +25,7 @@ import {resolveBaseUrlForEnqueueLinksFiltering,validators,SessionError,+ BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES,} from '@crawlee/basic';import type {BrowserController,@@ -37,7 +38,7 @@ import type {} from '@crawlee/browser-pool';import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool';import type { Cookie as CookieObject } from '@crawlee/types';-import { RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';+import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';import ow from 'ow';import type { BrowserLaunchContext } from './browser-launcher';@@ -440,17 +441,31 @@ export abstract class BrowserCrawler<}}+ private async containsSelectors(page: CommonPage, selectors: string[]): Promise<boolean> {+ return (await Promise.all(+ selectors.map((selector) => (page as any).$(selector)))+ ).some((el: any) => el !== null);+ }+protected override async isRequestBlocked(crawlingContext: Context): Promise<boolean> {const { page, response } = crawlingContext;+ // eslint-disable-next-line dot-notation+ const blockedStatusCodes = ((this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0)+ // eslint-disable-next-line dot-notation+ ? this.sessionPool!['blockedStatusCodes']+ : DEFAULT_BLOCKED_STATUS_CODES;+// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.- if (response?.status() === 403) {+ if (await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS) && response?.status() === 403) {await sleep(5000);++ // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.+ return this.containsSelectors(page, RETRY_CSS_SELECTORS);}- return (- await Promise.all(RETRY_CSS_SELECTORS.map((selector) => (page as any).$(selector)))- ).some((el) => el !== null);+ return await this.containsSelectors(page, RETRY_CSS_SELECTORS)+ || blockedStatusCodes.includes(response?.status() ?? 0);}/**diff --git a/packages/core/src/session_pool/consts.ts b/packages/core/src/session_pool/consts.tsnew file mode 100644index 000000000..8fdfe50d6--- /dev/null+++ b/packages/core/src/session_pool/consts.ts@@ -0,0 +1,3 @@+export const BLOCKED_STATUS_CODES = [401, 403, 429];+export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';+export const MAX_POOL_SIZE = 1000;diff --git a/packages/core/src/session_pool/index.ts b/packages/core/src/session_pool/index.tsindex 5d387f855..eaedabfa4 100644--- a/packages/core/src/session_pool/index.ts+++ b/packages/core/src/session_pool/index.ts@@ -2,3 +2,4 @@ export * from './errors';export * from './events';export * from './session';export * from './session_pool';+export * from './consts';diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex 64fad7ee7..21b6c4825 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -5,6 +5,7 @@ import type { Dictionary } from '@crawlee/types';import { AsyncQueue } from '@sapphire/async-queue';import ow from 'ow';+import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts';import type { SessionOptions } from './session';import { Session } from './session';import { Configuration } from '../configuration';@@ -159,12 +160,12 @@ export class SessionPool extends EventEmitter {}));const {- maxPoolSize = 1000,+ maxPoolSize = MAX_POOL_SIZE,persistStateKeyValueStoreId,- persistStateKey = 'SDK_SESSION_POOL_STATE',+ persistStateKey = PERSIST_STATE_KEY,createSessionFunction,sessionOptions = {},- blockedStatusCodes = [401, 403, 429],+ blockedStatusCodes = BLOCKED_STATUS_CODES,log = defaultLog,} = options;diff --git a/packages/utils/src/internals/blocked.ts b/packages/utils/src/internals/blocked.tsindex be78a32aa..f28c30407 100644--- a/packages/utils/src/internals/blocked.ts+++ b/packages/utils/src/internals/blocked.ts@@ -1,8 +1,12 @@+export const CLOUDFLARE_RETRY_CSS_SELECTORS = [+ 'iframe[src^="https://challenges.cloudflare.com"]',+];+/*** CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.*/export const RETRY_CSS_SELECTORS = [- 'iframe[src^="https://challenges.cloudflare.com"]',+ ...CLOUDFLARE_RETRY_CSS_SELECTORS,'div#infoDiv0 a[href*="//www.google.com/policies/terms/"]','iframe[src*="_Incapsula_Resource"]',];diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 9abca64e3..1f1f63692 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -3,6 +3,7 @@ import type { Server } from 'http';import { ENV_VARS } from '@apify/consts';import log from '@apify/log';import { BROWSER_POOL_EVENTS, BrowserPool, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';+import { BLOCKED_STATUS_CODES } from '@crawlee/core';import type {PuppeteerCrawlingContext,PuppeteerGoToOptions,@@ -488,7 +489,7 @@ describe('BrowserCrawler', () => {test('should throw on "blocked" status codes', async () => {const baseUrl = 'https://example.com/';- const sources = [401, 403, 429].map((statusCode) => {+ const sources = BLOCKED_STATUS_CODES.map((statusCode) => {return {url: baseUrl + statusCode,userData: { statusCode },@@ -529,9 +530,77 @@ describe('BrowserCrawler', () => {expect(called).toBe(false);});+ test('retryOnBlocked should retry on Cloudflare challenge', async () => {+ const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];+ const maxSessionRotations = 1;++ let processed = false;+ const errorMessages: string[] = [];++ const crawler = new BrowserCrawlerTest({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ },+ retryOnBlocked: true,+ maxSessionRotations,+ requestHandler: async ({ page, response }) => {+ processed = true;+ },+ failedRequestHandler: async ({ request }) => {+ errorMessages.push(...request.errorMessages);+ },+ });++ await crawler.run(urls);++ expect(errorMessages).toHaveLength(urls.length * (maxSessionRotations + 1));+ expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);+ expect(processed).toBe(false);+ });++ test('retryOnBlocked throws on "blocked" status codes', async () => {+ const baseUrl = 'https://example.com/';+ const sources = BLOCKED_STATUS_CODES.map((statusCode) => {+ return {+ url: baseUrl + statusCode,+ userData: { statusCode },+ };+ });+ const requestList = await RequestList.open(null, sources);+ const maxSessionRotations = 1;+ const errorMessages: string[] = [];++ let processed = false;+ const crawler = new BrowserCrawlerTest({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ },+ requestList,+ retryOnBlocked: true,+ maxSessionRotations,+ requestHandler: async () => {+ processed = true;+ },+ failedRequestHandler: async ({ request }) => {+ errorMessages.push(...request.errorMessages);+ },+ });++ // @ts-expect-error Overriding protected method+ crawler._navigationHandler = async ({ request }) => {+ return { status: () => request.userData.statusCode };+ };++ await crawler.run();++ expect(errorMessages.length).toBe(sources.length * (maxSessionRotations + 1));+ expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);+ expect(processed).toBe(false);+ });+test('should throw on "blocked" status codes (retire session)', async () => {const baseUrl = 'https://example.com/';- const sources = [401, 403, 429].map((statusCode) => {+ const sources = BLOCKED_STATUS_CODES.map((statusCode) => {return {url: baseUrl + statusCode,userData: { statusCode },diff --git a/test/shared/_helper.ts b/test/shared/_helper.tsindex f141d738a..b4553814f 100644--- a/test/shared/_helper.ts+++ b/test/shared/_helper.ts@@ -103,6 +103,65 @@ body {console.log('Hello world!');`,},+ cloudflareBlocking: `+<!DOCTYPE html>+<head>+ <title>Just a moment...</title>+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">+ <meta http-equiv="X-UA-Compatible" content="IE=Edge">+ <meta name="robots" content="noindex,nofollow">+ <meta name="viewport" content="width=device-width,initial-scale=1">+ <link href="/cdn-cgi/styles/challenges.css" rel="stylesheet">+ <meta http-equiv="refresh" content="375">+ <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1?ray=837fec16d9534138"></script><script src="https://challenges.cloudflare.com/turnstile/v0/g/74bd6362/api.js?onload=AudPIu1&render=explicit" async="" defer="" crossorigin="anonymous"></script>+</head>+<body class="no-js">+ <div class="main-wrapper" role="main">+ <div class="main-content">+ <h1 class="zone-name-title h1">dummypage.co</h1>+ <h2 id="challenge-running" class="h2">Checking if the site connection is secure</h2>+ <div id="challenge-stage" style="display: flex;">+ <div id="turnstile-wrapper" class="captcha-prompt spacer">+ <div><iframe style="border: medium; overflow: hidden; width: 300px; height: 65px;" src="https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/g/turnstile/if/ov2/av0/rcv0/0/o9un1/0x4AAAAAAADnPIDROrmt1Wwj/dark/normal" allow="cross-origin-isolated; fullscreen" sandbox="allow-same-origin allow-scripts allow-popups" id="cf-chl-widget-o9un1" tabindex="0" title="Widget containing a Cloudflare security challenge"></iframe><input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-o9un1_response"></div>+ </div>+ </div>+ <div id="challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">+ <div class="lds-ring">+ <div></div>+ <div></div>+ <div></div>+ <div></div>+ </div>+ </div>+ <div id="challenge-body-text" class="core-msg spacer">dummypage.co needs to review the security of your connection before proceeding.</div>+ <div id="challenge-success" style="display: none;">+ <div class="h2">+ <span class="icon-wrapper">+ </span>Connection is secure</div>+ <div class="core-msg spacer">Proceeding...</div>+ </div>+ <noscript>+ <div id="challenge-error-title">+ <div class="h2">+ <span class="icon-wrapper">+ <div class="heading-icon warning-icon"></div>+ </span>+ <span id="challenge-error-text">Enable JavaScript and cookies to continue</span>+ </div>+ </div>+ </noscript>+ </div>+ </div>+ <div class="footer" role="contentinfo">+ <div class="footer-inner">+ <div class="clearfix diagnostic-wrapper">+ <div class="ray-id">Ray ID: <code>837fec16d9534138</code></div>+ </div>+ <div class="text-center" id="footer-text">Performance & security by <a rel="noopener noreferrer" href="https://www.cloudflare.com?utm_source=challenge&utm_campaign=m" target="_blank">Cloudflare</a></div>+ </div>+ </div>+</body>+</html>`,};export async function runExampleComServer(): Promise<[Server, number]> {@@ -187,6 +246,13 @@ export async function runExampleComServer(): Promise<[Server, number]> {special.get('/resources', async (_req, res) => {res.type('html').send(responseSamples.resources);});++ special.get('/cloudflareBlocking', async (_req, res) => {+ res+ .type('html')+ .status(403)+ .send(responseSamples.cloudflareBlocking);+ });})();// "cacheable" site with one page, scripts and stylesheets