Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: detect the AWS WAF challenge and CAPTCHA with `retryOnBlocked` (#4184)

Mikhail Koviazineda7b9480c69
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index bdb6a9a91..3c05c6240 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -501,7 +501,9 @@ export interface BasicCrawlerOptions<
*
* Currently supports:
* - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
+ * - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
* - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
+ * - **Imperva Incapsula**
*/
retryOnBlocked?: boolean;
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 0dcd4d57e..e1908240d 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -37,6 +37,7 @@ import type { CommonPage } from '@crawlee/browser-pool';
import type { Awaitable, Cookie as CookieObject, Dictionary, IBrowserPool } from '@crawlee/types';
import {
assertBrowserPoolNotConfigured,
+ AWS_WAF_RETRY_CSS_SELECTORS,
CLOUDFLARE_RETRY_CSS_SELECTORS,
parseArgument,
RETRY_CSS_SELECTORS,
@@ -515,7 +516,15 @@ export abstract class BrowserCrawler<
}
async #containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {
- const foundSelectors = (await Promise.all(selectors.map((selector) => (page as any).$(selector))))
+ const query = async () => Promise.all(selectors.map((selector) => (page as any).$(selector)));
+
+ // a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
+ const elements = await query().catch(async () => {
+ await sleep(1000);
+ return query();
+ });
+
+ const foundSelectors = elements
.map((x, i) => [x, selectors[i]] as [any, string])
.filter(([x]) => x !== null)
.map(([, selector]) => selector);
@@ -523,20 +532,26 @@ export abstract class BrowserCrawler<
return foundSelectors.length > 0 ? foundSelectors : null;
}
- async #isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {
- const { page, response } = crawlingContext;
-
- // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
- if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
- await sleep(5000);
+ async #detectSelfResolvingChallenge({
+ page,
+ response,
+ }: BrowserCrawlingContext<Page, Response>): Promise<string | null> {
+ const statusCode = response?.status();
- // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
- const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
+ if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
+ return 'Cloudflare';
+ }
- if (!foundSelectors) return false;
- return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
+ if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
+ return 'AWS WAF';
}
+ return null;
+ }
+
+ async #isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {
+ const { page, response } = crawlingContext;
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
const statusCode = response?.status() ?? 0;
@@ -749,12 +764,48 @@ export abstract class BrowserCrawler<
}
}
- async #handleBlockedRequestByContent(crawlingContext: BrowserCrawlingContext<Page, Response>) {
- if (this.retryOnBlocked) {
- const error = await this.#isRequestBlocked(crawlingContext);
- if (error) throw new SessionError(error);
+ async #handleBlockedRequestByContent(
+ crawlingContext: BrowserCrawlingContext<Page, Response>,
+ ): Promise<{ response?: Response }> {
+ if (!this.retryOnBlocked) return {};
+
+ const page = crawlingContext.page as any;
+
+ // A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
+ let navigationResponse: Response | undefined;
+ const onResponse = (response: any) => {
+ if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
+ navigationResponse = response;
+ }
+ };
+
+ page.on('response', onResponse);
+
+ try {
+ // Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
+ // Wait 5 seconds for the JS challenge to load / resolve before judging the page.
+ const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
+
+ if (challenge) {
+ await sleep(5000);
+
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
+
+ if (foundSelectors) {
+ throw new SessionError(
+ `${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`,
+ );
+ }
+
+ return navigationResponse ? { response: navigationResponse } : {};
+ }
+ } finally {
+ page.off('response', onResponse);
}
+ const error = await this.#isRequestBlocked(crawlingContext);
+ if (error) throw new SessionError(error);
+
return {};
}
diff --git a/packages/utils/src/internals/blocked.ts b/packages/utils/src/internals/blocked.ts
index 07c2c922b..9e9210c50 100644
--- a/packages/utils/src/internals/blocked.ts
+++ b/packages/utils/src/internals/blocked.ts
@@ -1,10 +1,22 @@
export const CLOUDFLARE_RETRY_CSS_SELECTORS = ['#turnstile-wrapper iframe[src^="https://challenges.cloudflare.com"]'];
+/**
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
+ */
+export const AWS_WAF_RETRY_CSS_SELECTORS = [
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/challenge.js"]) > body > #challenge-container',
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/captcha.js"]) > body > #captcha-container',
+];
+
/**
* CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
*/
export const RETRY_CSS_SELECTORS = [
...CLOUDFLARE_RETRY_CSS_SELECTORS,
+ ...AWS_WAF_RETRY_CSS_SELECTORS,
'div#infoDiv0 a[href*="//www.google.com/policies/terms/"]',
'iframe[src*="_Incapsula_Resource"]',
];
diff --git a/test/core/crawlers/browser-crawler.test.ts b/test/core/crawlers/browser-crawler.test.ts
index 9ddd7836a..e05be2208 100644
--- a/test/core/crawlers/browser-crawler.test.ts
+++ b/test/core/crawlers/browser-crawler.test.ts
@@ -907,6 +907,66 @@ describe('BrowserCrawler', () => {
expect(processed).toBe(false);
});
+ test.concurrent('retryOnBlocked should retry on an AWS WAF challenge that ends in a CAPTCHA', async () => {
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ const urls = [new URL('/special/awsWafChallengeFailed', serverAddress).href];
+ const maxRequestRetries = 1;
+
+ let processed = false;
+ const errorMessages: string[] = [];
+
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ retryOnBlocked: true,
+ maxRequestRetries,
+ requestHandler: async () => {
+ processed = true;
+ },
+ failedRequestHandler: async ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
+
+ await crawler.run(urls);
+
+ expect(errorMessages).toHaveLength(urls.length * (maxRequestRetries + 1));
+ expect(
+ errorMessages.every((x) => x.includes('AWS WAF challenge failed') && x.includes('#captcha-container')),
+ ).toBe(true);
+ expect(processed).toBe(false);
+ });
+
+ test.concurrent('retryOnBlocked should wait for an AWS WAF challenge to resolve', async () => {
+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
+
+ const urls = [new URL('/special/awsWafChallengePassed', serverAddress).href];
+
+ const handled: { title: string; status?: number }[] = [];
+ const errorMessages: string[] = [];
+
+ const crawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ retryOnBlocked: true,
+ maxRequestRetries: 0,
+ requestHandler: async ({ page, response }) => {
+ handled.push({ title: await page.title(), status: response?.status() });
+ },
+ failedRequestHandler: async ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
+
+ await crawler.run(urls);
+
+ expect(errorMessages).toHaveLength(0);
+ expect(handled).toStrictEqual([{ title: 'Example Domain', status: 200 }]);
+ });
+
test.concurrent('retryOnBlocked throws on "blocked" status codes', async () => {
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
diff --git a/test/core/crawlers/http-crawler.test.ts b/test/core/crawlers/http-crawler.test.ts
index 3864e0a0f..dab3b6ae9 100644
--- a/test/core/crawlers/http-crawler.test.ts
+++ b/test/core/crawlers/http-crawler.test.ts
@@ -103,6 +103,69 @@ router.set('/meta-charset-html5', (req, res) => {
res.end(iconv.encode(html, 'windows-1250'));
});
+// the AWS WAF challenge interstitial, as served in October 2026: a `202` with an empty title and no content
+router.set('/awsWafChallenge', (req, res) => {
+ res.statusCode = 202;
+ res.setHeader('content-type', 'text/html; charset=UTF-8');
+ res.setHeader('x-amzn-waf-action', 'challenge');
+ res.end(`<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title></title>
+ <script type="text/javascript">
+ window.awsWafCookieDomainList = ['example.com'];
+ window.gokuProps = { "key": "AQIDAHjcYu", "iv": "CgAGUjJldAAABiGQ", "context": "loYdKdld00dlbxeUxL9OCW1a9K3GDRIu" };
+ </script>
+ <script src="https://fb423e1ef94f.6277d64d.us-east-1.token.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/challenge.js"></script>
+</head>
+<body>
+ <div id="challenge-container"></div>
+ <noscript>
+ <h1>JavaScript is disabled</h1>
+ In order to continue, we need to verify that you're not a robot.
+ </noscript>
+</body>
+</html>`);
+});
+
+// the CAPTCHA a failed AWS WAF challenge escalates to, served with a `405`
+router.set('/awsWafCaptcha', (req, res) => {
+ res.statusCode = 405;
+ res.setHeader('content-type', 'text/html; charset=UTF-8');
+ res.setHeader('x-amzn-waf-action', 'captcha');
+ res.end(`<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>Human Verification</title>
+ <script src="https://fb423e1ef94f.6277d64d.us-east-1.token.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/challenge.js"></script>
+ <script src="https://fb423e1ef94f.6277d64d.us-east-1.captcha.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/captcha.js"></script>
+</head>
+<body>
+ <div id="captcha-container"></div>
+</body>
+</html>`);
+});
+
+// a regular page of a site that loads the AWS WAF integration scripts and embeds the CAPTCHA into its own form
+router.set('/awsWafIntegration', (req, res) => {
+ res.setHeader('content-type', 'text/html; charset=UTF-8');
+ res.end(`<!DOCTYPE html>
+<html lang="en">
+<head>
+ <title>Example Domain</title>
+ <script src="https://fb423e1ef94f.us-east-1.sdk.awswaf.com/fb423e1ef94f/c3382d439950/challenge.js" defer></script>
+ <script src="https://fb423e1ef94f.us-east-1.captcha.awswaf.com/fb423e1ef94f/c3382d439950/captcha.js" defer></script>
+</head>
+<body>
+ <div id="content">
+ <form><div id="captcha-container"></div><div id="challenge-container"></div></form>
+ </div>
+</body>
+</html>`);
+});
+
let server: http.Server;
let url: string;
@@ -564,6 +627,54 @@ test('should retry on 403 even with disallowed content-type', async () => {
expect(succeeded[0].retryCount).toBe(1);
});
+test.each([
+ ['challenge served with a 202 status', '/awsWafChallenge', '#challenge-container'],
+ ['CAPTCHA served with a 405 status', '/awsWafCaptcha', '#captcha-container'],
+])('retryOnBlocked detects the AWS WAF %s', async (_name, path, container) => {
+ const succeeded: any[] = [];
+ const errorMessages: string[] = [];
+
+ const crawler = new HttpCrawler({
+ maxConcurrency: 1,
+ maxRequestRetries: 1,
+ retryOnBlocked: true,
+ requestHandler: async ({ request }) => {
+ succeeded.push(request);
+ },
+ failedRequestHandler: async ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
+
+ await crawler.run([`${url}${path}`]);
+
+ expect(succeeded).toHaveLength(0);
+ expect(errorMessages).toHaveLength(2);
+ expect(errorMessages.every((x) => x.includes('Found selectors') && x.includes(container))).toBe(true);
+});
+
+test('retryOnBlocked lets through a regular page that loads the AWS WAF integration scripts', async () => {
+ const succeeded: any[] = [];
+ const failed: any[] = [];
+
+ const crawler = new HttpCrawler({
+ maxConcurrency: 1,
+ maxRequestRetries: 0,
+ retryOnBlocked: true,
+ requestHandler: async ({ request }) => {
+ succeeded.push(request);
+ },
+ failedRequestHandler: async ({ request }) => {
+ failed.push(request);
+ },
+ });
+
+ await crawler.run([`${url}/awsWafIntegration`]);
+
+ expect(succeeded).toHaveLength(1);
+ expect(failed).toHaveLength(0);
+});
+
test('navigation hooks can override context members via return value', async () => {
let observedBody: string | undefined;
let observedStatus: number | undefined;
diff --git a/test/core/crawlers/playwright-crawler.test.ts b/test/core/crawlers/playwright-crawler.test.ts
index 5a45c0c32..1a1e5834d 100644
--- a/test/core/crawlers/playwright-crawler.test.ts
+++ b/test/core/crawlers/playwright-crawler.test.ts
@@ -21,7 +21,7 @@ import { z } from 'zod';
import log from '@apify/log';
-import { startExpressAppPromise } from '../../shared/_helper.js';
+import { runExampleComServer, startExpressAppPromise } from '../../shared/_helper.js';
if (os.platform() === 'win32') vitest.setConfig({ testTimeout: 2 * 60 * 1e3 });
@@ -135,6 +135,89 @@ describe('PlaywrightCrawler', () => {
});
});
+ describe('retryOnBlocked', () => {
+ let exampleComServer: Server;
+ let exampleComUrl: string;
+
+ beforeAll(async () => {
+ let exampleComPort: number;
+ [exampleComServer, exampleComPort] = await runExampleComServer();
+ exampleComUrl = `http://${HOSTNAME}:${exampleComPort}`;
+ });
+
+ afterAll(() => {
+ exampleComServer.close();
+ });
+
+ test.each(['chromium', 'firefox'] as const)(
+ 'waits for an AWS WAF challenge to resolve with %s',
+ async (browser) => {
+ const handled: { title: string; status?: number; body?: string }[] = [];
+ const errorMessages: string[] = [];
+
+ const crawler = new PlaywrightCrawler({
+ browserPool: playwrightBrowserPool({
+ launchContext: { launcher: playwright[browser] },
+ useFingerprints: false,
+ }),
+ retryOnBlocked: true,
+ maxRequestRetries: 0,
+ requestHandler: async ({ page, response }) => {
+ handled.push({
+ title: await page.title(),
+ status: response?.status(),
+ body: await response?.text(),
+ });
+ },
+ failedRequestHandler: ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
+
+ await crawler.run([`${exampleComUrl}/special/awsWafChallengePassed`]);
+
+ expect(errorMessages).toHaveLength(0);
+ expect(handled).toStrictEqual([
+ {
+ title: 'Example Domain',
+ status: 200,
+ body: expect.stringContaining('<title>Example Domain</title>'),
+ },
+ ]);
+ },
+ );
+
+ test.each(['chromium', 'firefox'] as const)(
+ 'retries an AWS WAF challenge that ends in a CAPTCHA with %s',
+ async (browser) => {
+ let processed = false;
+ const errorMessages: string[] = [];
+
+ const crawler = new PlaywrightCrawler({
+ browserPool: playwrightBrowserPool({
+ launchContext: { launcher: playwright[browser] },
+ useFingerprints: false,
+ }),
+ retryOnBlocked: true,
+ maxRequestRetries: 0,
+ requestHandler: async () => {
+ processed = true;
+ },
+ failedRequestHandler: ({ request }) => {
+ errorMessages.push(...request.errorMessages);
+ },
+ });
+
+ await crawler.run([`${exampleComUrl}/special/awsWafChallengeFailed`]);
+
+ expect(processed).toBe(false);
+ expect(errorMessages).toHaveLength(1);
+ expect(errorMessages[0]).toContain('AWS WAF challenge failed');
+ expect(errorMessages[0]).toContain('#captcha-container');
+ },
+ );
+ });
+
// https://github.com/apify/crawlee/issues/3670
test('should not silently drop requests when BrowserPool.newPage() times out', async () => {
const success: string[] = [];
diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
index 24a40279a..23dba2473 100644
--- a/test/shared/_helper.ts
+++ b/test/shared/_helper.ts
@@ -114,6 +114,50 @@ console.log('Hello world!');
<body>
<a href="/special/redirect-outside">click me</a>
</body>
+</html>`,
+ // the real scripts live on https://<id>.<id>.<region>.token.awswaf.com and .captcha.awswaf.com,
+ // the local paths keep the tests off the network
+ awsWafChallenge: (bodyScript: string) => `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title></title>
+ <script type="text/javascript">
+ window.awsWafCookieDomainList = ['dummypage.co'];
+ window.gokuProps = { "key": "AQIDAHjcYu", "iv": "CgAGUjJldAAABiGQ", "context": "loYdKdld00dlbxeUxL9OCW1a9K3GDRIu" };
+ </script>
+ <script src="/special/fb423e1ef94f.6277d64d.us-east-1.token.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/challenge.js"></script>
+</head>
+<body>
+ <div id="challenge-container"></div>
+ ${bodyScript}
+ <noscript>
+ <h1>JavaScript is disabled</h1>
+ In order to continue, we need to verify that you're not a robot.
+ </noscript>
+</body>
+</html>`,
+ awsWafCaptcha: `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>Human Verification</title>
+ <script type="text/javascript">
+ window.awsWafCookieDomainList = ['dummypage.co'];
+ window.gokuProps = { "key": "AQIDAHjcYu", "iv": "CgAGcDJmLAAABfB7", "context": "loYdKdld00dlbxeUxL9OCW1a9K3GDRIu" };
+ </script>
+ <script src="/special/fb423e1ef94f.6277d64d.us-east-1.token.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/challenge.js"></script>
+ <script src="/special/fb423e1ef94f.6277d64d.us-east-1.captcha.awswaf.com/fb423e1ef94f/c3382d439950/916d943f6a58/captcha.js"></script>
+</head>
+<body>
+ <div id="captcha-container"></div>
+ <noscript>
+ <h1>JavaScript is disabled</h1>
+ In order to continue, you need to verify that you're not a robot by solving a CAPTCHA puzzle.
+ </noscript>
+</body>
</html>`,
cloudflareBlocking: `
<!DOCTYPE html>
@@ -318,6 +362,22 @@ export async function runExampleComServer(): Promise<[Server, number]> {
res.type('html').status(403).send(responseSamples.cloudflareBlocking);
});
+ special.get('/awsWafChallengePassed', async (_req, res) => {
+ // a passed challenge reloads the page, which then gets the real content
+ const pass = `<script>setTimeout(() => window.location.replace('/special/html-type'), 500);</script>`;
+ res.type('html').status(202).send(responseSamples.awsWafChallenge(pass));
+ });
+
+ special.get('/awsWafChallengeFailed', async (_req, res) => {
+ // a failed challenge reloads the page as well, and gets the CAPTCHA instead
+ const fail = `<script>setTimeout(() => window.location.replace('/special/awsWafCaptcha'), 500);</script>`;
+ res.type('html').status(202).send(responseSamples.awsWafChallenge(fail));
+ });
+
+ special.get('/awsWafCaptcha', async (_req, res) => {
+ res.type('html').status(405).send(responseSamples.awsWafCaptcha);
+ });
+
special.get('/outside-iframe', (_req, res) => {
res.type('html').send(responseSamples.outsideIframe);
});