Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: more debug logs for checking enqueue links strategy (#2293)
Vlad Franguff674568bb8b
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex b4af68f44..6b29e7bd4 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -538,7 +538,10 @@ export abstract class BrowserCrawler<}if (!this.requestMatchesEnqueueStrategy(request)) {- this.log.debug(`Skipping request ${request.id} (${request.url}) because it does not match the enqueue strategy.`);+ this.log.debug(+ // eslint-disable-next-line max-len, dot-notation+ `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`,+ );request.noRetry = true;request.state = RequestState.SKIPPED;diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 47a68ee13..5549d0925 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -481,7 +481,10 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hrequest.loadedUrl = response.url;if (!this.requestMatchesEnqueueStrategy(request)) {- this.log.debug(`Skipping request ${request.id} (${request.url}) because it does not match the enqueue strategy.`);+ this.log.debug(+ // eslint-disable-next-line max-len, dot-notation+ `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`,+ );request.noRetry = true;request.state = RequestState.SKIPPED;