Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Make log message in RequestQueue.isFinished more accurate (#2848)

Jan Buchar3d124aee8f6f
diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.ts
index 1787de974..bf23021f0 100644
--- a/packages/core/src/storages/request_queue_v2.ts
+++ b/packages/core/src/storages/request_queue_v2.ts
@@ -60,6 +60,7 @@ export class RequestQueue extends RequestProvider {
private listHeadAndLockPromise: Promise<void> | null = null;
private queueHasLockedRequests: boolean | undefined = undefined;
private shouldCheckForForefrontRequests = false;
+ private dequeuedRequestCount = 0;
constructor(options: RequestProviderOptions, config = Configuration.getGlobalConfig()) {
super(
@@ -179,9 +180,19 @@ export class RequestQueue extends RequestProvider {
return null;
}
+ this.dequeuedRequestCount += 1;
+
return request;
}
+ /**
+ * @inheritDoc
+ */
+ override async markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null> {
+ this.dequeuedRequestCount -= 1;
+ return await super.markRequestHandled(request);
+ }
+
/**
* @inheritDoc
*/
@@ -207,7 +218,11 @@ export class RequestQueue extends RequestProvider {
// We could not lock any new requests - decide based on whether the queue contains requests locked by another client
if (this.queueHasLockedRequests !== undefined) {
// The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much.
- if (this.queueHasLockedRequests && ++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {
+ if (
+ this.queueHasLockedRequests &&
+ this.dequeuedRequestCount === 0 &&
+ ++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0
+ ) {
this.log.info('The queue still contains requests locked by another client');
}