Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Make log message in RequestQueue.isFinished more accurate (#2848)
Jan Buchar3d124aee8f6f
diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.tsindex 1787de974..bf23021f0 100644--- a/packages/core/src/storages/request_queue_v2.ts+++ b/packages/core/src/storages/request_queue_v2.ts@@ -60,6 +60,7 @@ export class RequestQueue extends RequestProvider {private listHeadAndLockPromise: Promise<void> | null = null;private queueHasLockedRequests: boolean | undefined = undefined;private shouldCheckForForefrontRequests = false;+ private dequeuedRequestCount = 0;constructor(options: RequestProviderOptions, config = Configuration.getGlobalConfig()) {super(@@ -179,9 +180,19 @@ export class RequestQueue extends RequestProvider {return null;}+ this.dequeuedRequestCount += 1;+return request;}+ /**+ * @inheritDoc+ */+ override async markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null> {+ this.dequeuedRequestCount -= 1;+ return await super.markRequestHandled(request);+ }+/*** @inheritDoc*/@@ -207,7 +218,11 @@ export class RequestQueue extends RequestProvider {// We could not lock any new requests - decide based on whether the queue contains requests locked by another clientif (this.queueHasLockedRequests !== undefined) {// The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much.- if (this.queueHasLockedRequests && ++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {+ if (+ this.queueHasLockedRequests &&+ this.dequeuedRequestCount === 0 &&+ ++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0+ ) {this.log.info('The queue still contains requests locked by another client');}