Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: investigate and temp fix for possible 0-concurrency bug in RQv2 (#2494)
Vlad Frangu4ebe820573b2
diff --git a/biome.json b/biome.jsonindex db6b76e1a..9bd15d91f 100644--- a/biome.json+++ b/biome.json@@ -19,5 +19,8 @@"quoteProperties": "preserve","lineEnding": "lf"}+ },+ "linter": {+ "enabled": false}}diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.tsindex 6e9208986..59735c61f 100644--- a/packages/core/src/storages/request_provider.ts+++ b/packages/core/src/storages/request_provider.ts@@ -54,6 +54,8 @@ export abstract class RequestProvider implements IStorage {protected queuePausedForMigration = false;+ protected lastActivity = new Date();+constructor(options: InternalRequestProviderOptions,readonly config = Configuration.getGlobalConfig(),@@ -114,6 +116,8 @@ export abstract class RequestProvider implements IStorage {): Promise<RequestQueueOperationInfo> {checkStorageAccess();+ this.lastActivity = new Date();+ow(requestLike, ow.object);ow(options,@@ -193,6 +197,8 @@ export abstract class RequestProvider implements IStorage {): Promise<BatchAddRequestsResult> {checkStorageAccess();+ this.lastActivity = new Date();+ow(requestsLike, ow.array);ow(options,@@ -308,6 +314,8 @@ export abstract class RequestProvider implements IStorage {): Promise<AddRequestsBatchedResult> {checkStorageAccess();+ this.lastActivity = new Date();+ow(options,ow.object.exactShape({@@ -464,6 +472,8 @@ export abstract class RequestProvider implements IStorage {async markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null> {checkStorageAccess();+ this.lastActivity = new Date();+ow(request,ow.object.partialShape({@@ -512,6 +522,8 @@ export abstract class RequestProvider implements IStorage {): Promise<RequestQueueOperationInfo | null> {checkStorageAccess();+ this.lastActivity = new Date();+ow(request,ow.object.partialShape({@@ -566,13 +578,51 @@ export abstract class RequestProvider implements IStorage {* but it will never return a false positive.*/async isFinished(): Promise<boolean> {- if (this.queueHeadIds.length() > 0 || this.inProgressCount() > 0) return false;+ // TODO: once/if we figure out why sometimes request queues get stuck (if it's even request queues), remove this once and for all :)+ if (Date.now() - +this.lastActivity > this.internalTimeoutMillis) {+ const message = `The request queue seems to be stuck for ${+ this.internalTimeoutMillis / 1000+ }s, resetting internal state.`;++ this.log.warning(message, {+ inProgress: [...this.inProgress],+ queueHeadIdsPending: this.queueHeadIds.length(),+ });++ // We only need to reset these two variables, no need to reset all the other stats+ this.queueHeadIds.clear();+ this.inProgress.clear();+ }++ if (this.queueHeadIds.length() > 0) {+ this.log.debug('There are still ids in the queue head that are pending processing', {+ queueHeadIdsPending: this.queueHeadIds.length(),+ });++ return false;+ }++ if (this.inProgressCount() > 0) {+ this.log.debug('There are still requests in progress (or zombie)', {+ inProgress: [...this.inProgress],+ });++ return false;+ }const currentHead = await this.client.listHead({ limit: 2 });++ if (currentHead.items.length !== 0) {+ this.log.debug(+ 'Queue head still returned requests that need to be processed (or that are locked by other clients)',+ );+ }+return currentHead.items.length === 0 && this.inProgressCount() === 0;}protected _reset() {+ this.lastActivity = new Date();this.queueHeadIds.clear();this.inProgress.clear();this.recentlyHandledRequestsCache.clear();diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 861fdc11d..6331f2954 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -83,8 +83,6 @@ class RequestQueue extends RequestProvider {hadMultipleClients?: boolean;}> | null = null;- private lastActivity = new Date();-/*** @internal*/@@ -120,6 +118,8 @@ class RequestQueue extends RequestProvider {override async fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null> {checkStorageAccess();+ this.lastActivity = new Date();+await this.ensureHeadIsNonEmpty();const nextRequestId = this.queueHeadIds.removeFirst();@@ -313,34 +313,6 @@ class RequestQueue extends RequestProvider {return isHeadConsistent && this.queueHeadIds.length() === 0 && this.inProgressCount() === 0;}- override async addRequest(...args: Parameters<RequestProvider['addRequest']>) {- checkStorageAccess();-- this.lastActivity = new Date();- return super.addRequest(...args);- }-- override async addRequests(...args: Parameters<RequestProvider['addRequests']>) {- checkStorageAccess();-- this.lastActivity = new Date();- return super.addRequests(...args);- }-- override async addRequestsBatched(...args: Parameters<RequestProvider['addRequestsBatched']>) {- checkStorageAccess();-- this.lastActivity = new Date();- return super.addRequestsBatched(...args);- }-- override async markRequestHandled(...args: Parameters<RequestProvider['markRequestHandled']>) {- checkStorageAccess();-- this.lastActivity = new Date();- return super.markRequestHandled(...args);- }-/*** Reclaims a failed request back to the queue, so that it can be returned for processing later again* by another call to {@apilink RequestQueue.fetchNextRequest}.@@ -350,8 +322,6 @@ class RequestQueue extends RequestProvider {override async reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>) {checkStorageAccess();- this.lastActivity = new Date();-const [request, options] = args;const forefront = options?.forefront ?? false;@@ -376,11 +346,6 @@ class RequestQueue extends RequestProvider {return result;}- protected override _reset() {- super._reset();- this.lastActivity = new Date();- }-/*** Opens a request queue and returns a promise resolving to an instance* of the {@apilink RequestQueue} class.diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.tsindex 136256325..4f4f12484 100644--- a/packages/core/src/storages/request_queue_v2.ts+++ b/packages/core/src/storages/request_queue_v2.ts@@ -81,8 +81,6 @@ export class RequestQueue extends RequestProvider {* Caches information about request to beware of unneeded addRequest() calls.*/protected override _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void {- checkStorageAccess();-super._cacheRequest(cacheKey, queueOperationInfo);this.requestCache.remove(queueOperationInfo.requestId);@@ -102,6 +100,8 @@ export class RequestQueue extends RequestProvider {override async fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null> {checkStorageAccess();+ this.lastActivity = new Date();+await this.ensureHeadIsNonEmpty();const nextRequestId = this.queueHeadIds.removeFirst();@@ -171,8 +171,6 @@ export class RequestQueue extends RequestProvider {override async reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): ReturnType<RequestProvider['reclaimRequest']> {- checkStorageAccess();-const res = await super.reclaimRequest(...args);if (res) {@@ -215,8 +213,6 @@ export class RequestQueue extends RequestProvider {}private async _listHeadAndLock(): Promise<void> {- checkStorageAccess();-const headData = await this.client.listAndLockHead({ limit: 25, lockSecs: this.requestLockSecs });for (const { id, uniqueKey } of headData.items) {@@ -337,8 +333,6 @@ export class RequestQueue extends RequestProvider {}private async _prolongRequestLock(requestId: string): Promise<Date | null> {- checkStorageAccess();-try {const res = await this.client.prolongRequestLock(requestId, { lockSecs: this.requestLockSecs });return res.lockExpiresAt;@@ -356,8 +350,6 @@ export class RequestQueue extends RequestProvider {}protected override _reset() {- checkStorageAccess();-super._reset();this._listHeadAndLockPromise = null;}@@ -367,8 +359,6 @@ export class RequestQueue extends RequestProvider {}protected async _clearPossibleLocks() {- checkStorageAccess();-this.queuePausedForMigration = true;let requestId: string | null;