Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: log cause with `retryOnBlocked` (#2252)
Jindřich Bäre19a773693cf
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 2677338ca..db0bf38ae 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -751,7 +751,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn ROTATE_PROXY_ERRORS.some((x: string) => (this._getMessageFromError(error) as any)?.includes(x));}- protected isRequestBlocked(_crawlingContext: Context) {+ /**+ * Checks whether the given crawling context is getting blocked by anti-bot protection using several heuristics.+ * Returns `false` if the request is not blocked, otherwise returns a string with a description of the block reason.+ * @param _crawlingContext The crawling context to check.+ */+ protected async isRequestBlocked(_crawlingContext: Context): Promise<string | false> {throw new Error('the "isRequestBlocked" method is not implemented in this crawler.');}diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex f78ceaf7d..b4af68f44 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -441,13 +441,18 @@ export abstract class BrowserCrawler<}}- private async containsSelectors(page: CommonPage, selectors: string[]): Promise<boolean> {- return (await Promise.all(+ private async containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {+ const foundSelectors = (await Promise.all(selectors.map((selector) => (page as any).$(selector)))- ).some((el: any) => el !== null);+ )+ .map((x, i) => [x, selectors[i]] as [any, string])+ .filter(([x]) => x !== null)+ .map(([, selector]) => selector);++ return foundSelectors.length > 0 ? foundSelectors : null;}- protected override async isRequestBlocked(crawlingContext: Context): Promise<boolean> {+ protected override async isRequestBlocked(crawlingContext: Context): Promise<string | false> {const { page, response } = crawlingContext;// eslint-disable-next-line dot-notation@@ -461,11 +466,19 @@ export abstract class BrowserCrawler<await sleep(5000);// here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.- return this.containsSelectors(page, RETRY_CSS_SELECTORS);+ const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);++ if (!foundSelectors) return false;+ return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;}- return await this.containsSelectors(page, RETRY_CSS_SELECTORS)- || blockedStatusCodes.includes(response?.status() ?? 0);+ const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);+ const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));++ if (foundSelectors) return `Found selectors: ${foundSelectors.join(', ')}`;+ if (blockedStatusCode) return `Received blocked status code: ${blockedStatusCode}`;++ return false;}/**@@ -534,9 +547,8 @@ export abstract class BrowserCrawler<}if (this.retryOnBlocked) {- if (await this.isRequestBlocked(crawlingContext)) {- throw new SessionError();- }+ const error = await this.isRequestBlocked(crawlingContext);+ if (error) throw new SessionError(error);}request.state = RequestState.REQUEST_HANDLER;diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 15076503b..47a68ee13 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -500,8 +500,9 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H});}- if (this.retryOnBlocked && await this.isRequestBlocked(crawlingContext)) {- throw new SessionError();+ if (this.retryOnBlocked) {+ const error = await this.isRequestBlocked(crawlingContext);+ if (error) throw new SessionError(error);}request.state = RequestState.REQUEST_HANDLER;@@ -518,11 +519,15 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H}}- protected override async isRequestBlocked(crawlingContext: Context) {+ protected override async isRequestBlocked(crawlingContext: Context): Promise<string | false> {if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {const $ = await crawlingContext.parseWithCheerio();- return RETRY_CSS_SELECTORS.some((selector) => $(selector).length > 0);+ const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);++ if (foundSelectors.length > 0) {+ return `Found selectors: ${foundSelectors.join(', ')}`;+ }}return false;}