Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Simplified RequestQueueV2 implementation (#2775)
Jan Buchard1a094a47eae
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 0c87d8b36..25fc29909 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -671,9 +671,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`if (this.requestQueue) {this.requestQueue.internalTimeoutMillis = this.internalTimeoutMillis;- // for request queue v2, we want to lock requests by the timeout that would also account for internals (plus 5 seconds padding), but- // with a minimum of a minute- this.requestQueue.requestLockSecs = Math.max(this.internalTimeoutMillis / 1000 + 5, 60);+ // for request queue v2, we want to lock requests for slightly longer than the request handler timeout so that there is some padding for locking-related overhead,+ // but never for less than a minute+ this.requestQueue.requestLockSecs = Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60);}this.maxRequestRetries = maxRequestRetries;diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.tsindex 3be128e01..53983fbe2 100644--- a/packages/core/src/storages/request_provider.ts+++ b/packages/core/src/storages/request_provider.ts@@ -44,15 +44,6 @@ export abstract class RequestProvider implements IStorage {assumedTotalCount = 0;assumedHandledCount = 0;- /**- * Counts enqueued `forefront` requests. This is used to invalidate the local head cache.- * We can trust these numbers only in a case that queue is used by a single client.- *- * Note that this number does not have to be persisted, as the local head cache is dropped- * on a migration event.- */- protected assumedForefrontCount = 0;-private initialCount = 0;protected queueHeadIds = new ListDictionary<string>();@@ -66,6 +57,8 @@ export abstract class RequestProvider implements IStorage {protected isFinishedCalledWhileHeadWasNotEmpty = 0;+ protected inProgressRequestBatchCount = 0;+constructor(options: InternalRequestProviderOptions,readonly config = Configuration.getGlobalConfig(),@@ -135,7 +128,7 @@ export abstract class RequestProvider implements IStorage {const requests = await this._fetchRequestsFromUrl(requestLike as InternalSource);const processedRequests = await this._addFetchedRequests(requestLike as InternalSource, requests, options);- return processedRequests[0];+ return { ...processedRequests[0], forefront };}ow(@@ -160,11 +153,15 @@ export abstract class RequestProvider implements IStorage {wasAlreadyHandled: cachedInfo.isHandled,requestId: cachedInfo.id,uniqueKey: cachedInfo.uniqueKey,+ forefront,};}- const queueOperationInfo = (await this.client.addRequest(request, { forefront })) as RequestQueueOperationInfo;- queueOperationInfo.uniqueKey = request.uniqueKey;+ const queueOperationInfo = {+ ...(await this.client.addRequest(request, { forefront })),+ uniqueKey: request.uniqueKey,+ forefront,+ } satisfies RequestQueueOperationInfo;const { requestId, wasAlreadyPresent } = queueOperationInfo;this._cacheRequest(cacheKey, queueOperationInfo);@@ -172,8 +169,6 @@ export abstract class RequestProvider implements IStorage {if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {this.assumedTotalCount++;- this.assumedForefrontCount += forefront ? 1 : 0;-// Performance optimization: add request straight to head if possiblethis._maybeAddRequestToQueueHead(requestId, forefront);}@@ -275,7 +270,7 @@ export abstract class RequestProvider implements IStorage {// Report unprocessed requestsresults.unprocessedRequests = apiResults.unprocessedRequests;- // Add all new requests to the queue head+ // Add all new requests to the requestCachefor (const newRequest of apiResults.processedRequests) {// Add the new request to the processed listresults.processedRequests.push(newRequest);@@ -285,14 +280,12 @@ export abstract class RequestProvider implements IStorage {const { requestId, wasAlreadyPresent } = newRequest;if (cache) {- this._cacheRequest(cacheKey, newRequest);+ this._cacheRequest(cacheKey, { ...newRequest, forefront });}if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {this.assumedTotalCount++;- this.assumedForefrontCount += forefront ? 1 : 0;-// Performance optimization: add request straight to head if possiblethis._maybeAddRequestToQueueHead(requestId, forefront);}@@ -418,6 +411,11 @@ export abstract class RequestProvider implements IStorage {resolve(finalAddedRequests);});+ this.inProgressRequestBatchCount += 1;+ void promise.finally(() => {+ this.inProgressRequestBatchCount -= 1;+ });+// If the user wants to wait for all the requests to be added, we wait for the promise to resolve for themif (options.waitForAllRequestsToBeAdded) {addedRequests.push(...(await promise));@@ -485,13 +483,18 @@ export abstract class RequestProvider implements IStorage {}),);+ const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;+const handledAt = request.handledAt ?? new Date().toISOString();- const queueOperationInfo = (await this.client.updateRequest({- ...request,- handledAt,- })) as RequestQueueOperationInfo;+ const queueOperationInfo = {+ ...(await this.client.updateRequest({+ ...request,+ handledAt,+ })),+ uniqueKey: request.uniqueKey,+ forefront,+ } satisfies RequestQueueOperationInfo;request.handledAt = handledAt;- queueOperationInfo.uniqueKey = request.uniqueKey;this.recentlyHandledRequestsCache.add(request.id, true);@@ -538,11 +541,13 @@ export abstract class RequestProvider implements IStorage {// TODO: If request hasn't been changed since the last getRequest(),// we don't need to call updateRequest() and thus improve performance.- const queueOperationInfo = (await this.client.updateRequest(request, {+ const queueOperationInfo = {+ ...(await this.client.updateRequest(request, {+ forefront,+ })),+ uniqueKey: request.uniqueKey,forefront,- })) as RequestQueueOperationInfo;- queueOperationInfo.uniqueKey = request.uniqueKey;- this.assumedForefrontCount += forefront ? 1 : 0;+ } satisfies RequestQueueOperationInfo;this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);return queueOperationInfo;@@ -564,82 +569,10 @@ export abstract class RequestProvider implements IStorage {/*** Resolves to `true` if all requests were already handled and there are no more left.* Due to the nature of distributed storage used by the queue,- * the function might occasionally return a false negative,- * but it will never return a false positive.+ * the function may occasionally return a false negative,+ * but it shall never return a false positive.*/- async isFinished(): Promise<boolean> {- // TODO: once/if we figure out why sometimes request queues get stuck (if it's even request queues), remove this once and for all :)- if (Date.now() - this.lastActivity.getTime() > this.internalTimeoutMillis) {- const maybeHead = await this.client.listHead({ limit: 1 });-- const request = maybeHead.items[0] ? await this.client.getRequest(maybeHead.items[0].id) : null;-- const message = `The request queue hasn't had activity for ${- this.internalTimeoutMillis / 1000- }s, resetting internal state.`;-- this.log.warning(message, {- queueHeadIdsPending: this.queueHeadIds.length(),- hasPendingOrLockedRequests: maybeHead.items.length > 0,- pendingRequestIsLocked: request?.lockExpiresAt,- });-- this.queueHeadIds.clear();- // This cache may be the bane of our existence, but it's still required for v1...- this.recentlyHandledRequestsCache.clear();- }-- if (this.queueHeadIds.length() > 0) {- this.log.debug('There are still ids in the queue head that are pending processing', {- queueHeadIdsPending: this.queueHeadIds.length(),- });-- return false;- }-- const currentHead = await this.client.listHead({ limit: 2 });-- if (currentHead.items.length !== 0) {- // Give users some more concrete info as to why their crawlers seem to be "hanging" doing nothing while we're waiting because the queue is technically- // not empty. We decided that a queue with elements in its head but that are also locked shouldn't return true in this function.- // If that ever changes, this function might need a rewrite- // The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much. This is also a very specific path that most crawlers shouldn't hit- if (++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {- const requests = await Promise.all(- currentHead.items.map(async (item) => this.client.getRequest(item.id)),- );-- this.log.info(- `Queue head still returned requests that need to be processed (or that are locked by other clients)`,- {- requests: requests- .map((r) => {- if (!r) {- return null;- }-- return {- id: r.id,- lockExpiresAt: r.lockExpiresAt,- lockedBy: r.lockByClient,- };- })- .filter(Boolean),- clientKey: this.clientKey,- },- );- } else {- this.log.debug(- 'Queue head still returned requests that need to be processed (or that are locked by other clients)',- {- requestIds: currentHead.items.map((item) => item.id),- },- );- }- }-- return currentHead.items.length === 0;- }+ abstract isFinished(): Promise<boolean>;protected _reset() {this.lastActivity = new Date();@@ -663,6 +596,7 @@ export abstract class RequestProvider implements IStorage {uniqueKey: queueOperationInfo.uniqueKey,hydrated: null,lockExpiresAt: null,+ forefront: queueOperationInfo.forefront,});}@@ -844,6 +778,8 @@ declare class BuiltRequestProvider extends RequestProvider {): Promise<Request<T> | null>;protected override ensureHeadIsNonEmpty(): Promise<void>;++ override isFinished(): Promise<boolean>;}interface RequestLruItem {@@ -852,6 +788,7 @@ interface RequestLruItem {id: string;hydrated: Request | null;lockExpiresAt: number | null;+ forefront: boolean;}export interface RequestProviderOptions {@@ -909,6 +846,7 @@ export interface RequestQueueOperationOptions {*/export interface RequestQueueOperationInfo extends QueueOperationInfo {uniqueKey: string;+ forefront: boolean;}export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions {diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 24b3e6cbd..0b501465d 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -236,11 +236,13 @@ class RequestQueue extends RequestProvider {return;this.queueHeadIds.add(requestId, requestId, false);+ const forefront = this.requestCache.get(getRequestId(uniqueKey))?.forefront ?? false;this._cacheRequest(getRequestId(uniqueKey), {requestId,wasAlreadyHandled: false,wasAlreadyPresent: true,uniqueKey,+ forefront,});});@@ -316,6 +318,10 @@ class RequestQueue extends RequestProvider {this._reset();}+ if (this.inProgressRequestBatchCount > 0) {+ return false;+ }+if (this.queueHeadIds.length() > 0 || this.inProgressCount() > 0) return false;const isHeadConsistent = await this._ensureHeadIsNonEmpty(true);diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.tsindex d2c3bd11c..1787de974 100644--- a/packages/core/src/storages/request_queue_v2.ts+++ b/packages/core/src/storages/request_queue_v2.ts@@ -1,12 +1,16 @@-import type { Dictionary } from '@crawlee/types';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { checkStorageAccess } from './access_checking';-import type { RequestQueueOperationInfo, RequestProviderOptions } from './request_provider';+import type {+ RequestQueueOperationInfo,+ RequestProviderOptions,+ RequestQueueOperationOptions,+} from './request_provider';import { RequestProvider } from './request_provider';import { getRequestId } from './utils';import { Configuration } from '../configuration';import { EventType } from '../events';-import type { Request } from '../request';+import type { Request, Source } from '../request';// Double the limit of RequestQueue v1 (1_000_000) as we also store keyed by request.id, not just from uniqueKeyconst MAX_CACHED_REQUESTS = 2_000_000;@@ -53,18 +57,9 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000;* @category Sources*/export class RequestQueue extends RequestProvider {- private _listHeadAndLockPromise: Promise<void> | null = null;-- /**- * Returns `true` if there are any requests in the queue that were enqueued to the forefront.- *- * Can return false negatives, but never false positives.- * @returns `true` if there are `forefront` requests in the queue.- */- private async hasPendingForefrontRequests(): Promise<boolean> {- const queueInfo = await this.client.get();- return this.assumedForefrontCount > 0 && !queueInfo?.hadMultipleClients;- }+ private listHeadAndLockPromise: Promise<void> | null = null;+ private queueHasLockedRequests: boolean | undefined = undefined;+ private shouldCheckForForefrontRequests = false;constructor(options: RequestProviderOptions, config = Configuration.getGlobalConfig()) {super(@@ -100,17 +95,53 @@ export class RequestQueue extends RequestProvider {id: queueOperationInfo.requestId,isHandled: queueOperationInfo.wasAlreadyHandled,uniqueKey: queueOperationInfo.uniqueKey,+ forefront: queueOperationInfo.forefront,hydrated: null,lockExpiresAt: null,});}+ /**+ * @inheritDoc+ */+ override async addRequest(+ requestLike: Source,+ options: RequestQueueOperationOptions = {},+ ): Promise<RequestQueueOperationInfo> {+ const result = await super.addRequest(requestLike, options);+ if (!result.wasAlreadyPresent && options.forefront) {+ this.shouldCheckForForefrontRequests = true;+ }+ return result;+ }++ /**+ * @inheritDoc+ */+ override async addRequests(+ requestsLike: Source[],+ options: RequestQueueOperationOptions = {},+ ): Promise<BatchAddRequestsResult> {+ const result = await super.addRequests(requestsLike, options);+ for (const request of result.processedRequests) {+ if (!request.wasAlreadyPresent && options.forefront) {+ this.shouldCheckForForefrontRequests = true;+ break;+ }+ }+ return result;+ }+/*** @inheritDoc*/override async fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null> {checkStorageAccess();+ if (this.queuePausedForMigration) {+ return null;+ }+this.lastActivity = new Date();await this.ensureHeadIsNonEmpty();@@ -122,15 +153,6 @@ export class RequestQueue extends RequestProvider {return null;}- // This should never happen, but...- if (this.recentlyHandledRequestsCache.get(nextRequestId)) {- this.log.warning('Queue head returned a request that was recently marked as handled?!', {- nextRequestId,- recentlyHandled: !!this.recentlyHandledRequestsCache.get(nextRequestId),- });- return null;- }-const request: Request | null = await this.getOrHydrateRequest(nextRequestId);// NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:@@ -154,13 +176,90 @@ export class RequestQueue extends RequestProvider {// will not put the request again to queueHeadDict.if (request.handledAt) {this.log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });- this.recentlyHandledRequestsCache.add(nextRequestId, true);return null;}return request;}+ /**+ * @inheritDoc+ */+ override async isFinished(): Promise<boolean> {+ // We are not finished if we're still adding new requests in the background+ if (this.inProgressRequestBatchCount > 0) {+ return false;+ }++ // If the local queue head is non-empty, we don't need to query the "upstream" queue to know we are not finished yet+ if (this.queueHeadIds.length() > 0) {+ return false;+ }++ // Local queue head is empty - try to fetch and lock more requests+ await this.ensureHeadIsNonEmpty();++ // We managed to lock something - we are not finished+ if (this.queueHeadIds.length() > 0) {+ return false;+ }++ // We could not lock any new requests - decide based on whether the queue contains requests locked by another client+ if (this.queueHasLockedRequests !== undefined) {+ // The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much.+ if (this.queueHasLockedRequests && ++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {+ this.log.info('The queue still contains requests locked by another client');+ }++ return !this.queueHasLockedRequests;+ }++ // The following is a legacy algorithm for checking if the queue is finished. It is used only for request queue clients that do not provide the `queueHasLockedRequests` flag.++ const currentHead = await this.client.listHead({ limit: 2 });++ if (currentHead.items.length === 0) {+ return true;+ }++ // Give users some more concrete info as to why their crawlers seem to be "hanging" doing nothing while we're waiting because the queue is technically+ // not empty. We decided that a queue with elements in its head but that are also locked shouldn't return true in this function.+ // If that ever changes, this function might need a rewrite+ // The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much. This is also a very specific path that most crawlers shouldn't hit+ if (++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {+ const requests = await Promise.all(currentHead.items.map(async (item) => this.client.getRequest(item.id)));++ this.log.info(+ `Queue head still returned requests that need to be processed (or that are locked by other clients)`,+ {+ requests: requests+ .map((r) => {+ if (!r) {+ return null;+ }++ return {+ id: r.id,+ lockExpiresAt: r.lockExpiresAt,+ lockedBy: r.lockByClient,+ };+ })+ .filter(Boolean),+ clientKey: this.clientKey,+ },+ );+ } else {+ this.log.debug(+ 'Queue head still returned requests that need to be processed (or that are locked by other clients)',+ {+ requestIds: currentHead.items.map((item) => item.id),+ },+ );+ }++ return false;+ }+/*** @inheritDoc*/@@ -172,6 +271,10 @@ export class RequestQueue extends RequestProvider {if (res) {const [request, options] = args;+ if (options?.forefront) {+ this.shouldCheckForForefrontRequests = true;+ }+// Try to delete the request lock if possibletry {await this.client.deleteRequestLock(request.id!, { forefront: options?.forefront ?? false });@@ -192,67 +295,102 @@ export class RequestQueue extends RequestProvider {}// We want to fetch ahead of time to minimize dead time- if (this.queueHeadIds.length() > 1 && !(await this.hasPendingForefrontRequests())) {+ if (this.queueHeadIds.length() > 1 && !this.shouldCheckForForefrontRequests) {return;}- this._listHeadAndLockPromise ??= this._listHeadAndLock().finally(() => {- this._listHeadAndLockPromise = null;+ this.listHeadAndLockPromise ??= this._listHeadAndLock().finally(() => {+ this.listHeadAndLockPromise = null;});- await this._listHeadAndLockPromise;+ await this.listHeadAndLockPromise;+ }++ private async giveUpLock(id?: string, uniqueKey?: string) {+ if (id === undefined) {+ return;+ }++ try {+ await this.client.deleteRequestLock(id);+ } catch {+ this.log.debug('Failed to delete request lock', { id, uniqueKey });+ }}private async _listHeadAndLock(): Promise<void> {- const forefront = await this.hasPendingForefrontRequests();+ // Make a copy so that we can clear the flag only if the whole method executes after the flag was set+ // (i.e, it was not set in the middle of the execution of the method)+ const shouldCheckForForefrontRequests = this.shouldCheckForForefrontRequests;++ const limit = 25;const headData = await this.client.listAndLockHead({- limit: Math.min(forefront ? this.assumedForefrontCount : 25, 25),+ limit,lockSecs: this.requestLockSecs,});+ this.queueHasLockedRequests = headData.queueHasLockedRequests;+const headIdBuffer = [];+ const forefrontHeadIdBuffer = [];for (const { id, uniqueKey } of headData.items) {- // Queue head index might be behind the main table, so ensure we don't recycle requests- if (- !id ||- !uniqueKey ||- this.recentlyHandledRequestsCache.get(id) ||- // If we tried to read new forefront requests, but another client appeared in the meantime, we can't be sure we'll only read our requests.- // To retain the correct queue ordering, we rollback this head read.- (forefront && headData.hadMultipleClients)- ) {- this.log.debug(`Skipping request from queue head as it's potentially invalid or recently handled`, {- id,- uniqueKey,- recentlyHandled: !!this.recentlyHandledRequestsCache.get(id),- inconsistentForefrontRead: forefront && headData.hadMultipleClients,- });+ if (!id || !uniqueKey) {+ this.log.warning(+ `Skipping request from queue head as it's invalid. Please report this with the provided metadata!`,+ {+ id,+ uniqueKey,+ },+ );// Remove the lock from the request for now, so that it can be picked up later// This may/may not succeed, but that's fine- try {- await this.client.deleteRequestLock(id);- } catch {- // Ignore- }-+ await this.giveUpLock(id, uniqueKey);continue;}- headIdBuffer.push(id);- this.assumedForefrontCount = Math.max(0, this.assumedForefrontCount - 1);+ // If we remember that we added the request ourselves and we added it to the forefront,+ // we will put it to the beginning of the local queue head to preserve the expected order.+ // If we do not remember that, we will enqueue it normally.+ const forefront = this.requestCache.get(getRequestId(uniqueKey))?.forefront ?? false;+ if (forefront) {+ forefrontHeadIdBuffer.unshift(id);+ } else {+ headIdBuffer.push(id);+ }++ // Ensure that the request is cached locallythis._cacheRequest(getRequestId(uniqueKey), {requestId: id,uniqueKey,wasAlreadyPresent: true,wasAlreadyHandled: false,+ forefront,});}- for (const id of forefront ? headIdBuffer.reverse() : headIdBuffer) {- this.queueHeadIds.add(id, id, forefront);+ for (const id of headIdBuffer) {+ this.queueHeadIds.add(id, id, false);+ }++ for (const id of forefrontHeadIdBuffer) {+ this.queueHeadIds.add(id, id, true);+ }++ // Unlock and forget requests that would make the local queue head grow over the limit+ const toUnlock = [];+ while (this.queueHeadIds.length() > limit) {+ toUnlock.push(this.queueHeadIds.removeLast()!);+ }++ if (toUnlock.length > 0) {+ await Promise.all(toUnlock.map(async (id) => await this.giveUpLock(id)));+ }++ if (shouldCheckForForefrontRequests) {+ this.shouldCheckForForefrontRequests = false;}}@@ -293,6 +431,7 @@ export class RequestQueue extends RequestProvider {hydrated: hydratedRequest,isHandled: hydratedRequest.handledAt !== null,lockExpiresAt: prolongResult.getTime(),+ forefront: false,});return hydratedRequest;@@ -361,7 +500,8 @@ export class RequestQueue extends RequestProvider {protected override _reset() {super._reset();- this._listHeadAndLockPromise = null;+ this.listHeadAndLockPromise = null;+ this.queueHasLockedRequests = undefined;}protected override _maybeAddRequestToQueueHead() {diff --git a/packages/core/test/request-queue/request-queue-v2.test.ts b/packages/core/test/request-queue/request-queue-v2.test.tsindex f311ac94d..d3b0affdd 100644--- a/packages/core/test/request-queue/request-queue-v2.test.ts+++ b/packages/core/test/request-queue/request-queue-v2.test.ts@@ -24,31 +24,27 @@ vitest.setConfig({ restoreMocks: false });describe('RequestQueueV2#isFinished should use listHead instead of listAndLock', () => {let queue: RequestQueueV2;let clientListHeadSpy: MockInstance<typeof queue.client.listHead>;- let listHeadCallCount = 0;- let clientListAndLockHeadSpy: MockInstance<typeof queue.client.listAndLockHead>;- let listAndLockHeadCallCount = 0;- let lockResult: ListAndLockHeadResult;beforeAll(async () => {queue = await makeQueue('is-finished', 2);clientListHeadSpy = vitest.spyOn(queue.client, 'listHead');- clientListAndLockHeadSpy = vitest.spyOn(queue.client, 'listAndLockHead');});test('should return false if there are still requests in the queue', async () => {expect(await queue.isFinished()).toBe(false);- expect(clientListHeadSpy).toHaveBeenCalledTimes(++listHeadCallCount);});test('should return false even if all requests are locked', async () => {- lockResult = await queue.client.listAndLockHead({ lockSecs: 60 });-- expect(lockResult.items.length).toBe(2);- expect(clientListAndLockHeadSpy).toHaveBeenCalledTimes(++listAndLockHeadCallCount);+ queue.client.listAndLockHead = async (options) => ({+ lockSecs: options.lockSecs,+ queueModifiedAt: new Date(),+ limit: 10,+ items: [],+ queueHasLockedRequests: true,+ });expect(await queue.isFinished()).toBe(false);- expect(clientListHeadSpy).toHaveBeenCalledTimes(++listHeadCallCount);- expect(clientListAndLockHeadSpy).toHaveBeenCalledTimes(listAndLockHeadCallCount);+ expect(clientListHeadSpy).not.toHaveBeenCalled();});});@@ -81,7 +77,7 @@ describe('RequestQueueV2#isFinished should return true once locked requests areexpect(clientListHeadSpy).toHaveBeenCalledWith({ limit: 2 });expect(clientListHeadSpy).toHaveBeenCalledTimes(++listHeadCallCount);// One time- expect(clientListAndLockHeadSpy).toHaveBeenCalledTimes(1);+ expect(clientListAndLockHeadSpy).toHaveBeenCalled();});});diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.tsindex 4d2af1739..df6bd18b5 100644--- a/packages/types/src/storages.ts+++ b/packages/types/src/storages.ts@@ -227,6 +227,7 @@ export interface ListAndLockOptions extends ListOptions {export interface ListAndLockHeadResult extends QueueHead {lockSecs: number;+ queueHasLockedRequests?: boolean;}export interface ProlongRequestLockOptions {diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex 5d3b3ba5a..904e5c9e0 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -305,6 +305,7 @@ describe('RequestQueue remote', () => {uniqueKey: requestA.uniqueKey,wasAlreadyPresent: true,wasAlreadyHandled: false,+ forefront: false,});});@@ -335,6 +336,7 @@ describe('RequestQueue remote', () => {uniqueKey: requestX.uniqueKey,wasAlreadyPresent: true,wasAlreadyHandled: true,+ forefront: false,});});@@ -362,6 +364,7 @@ describe('RequestQueue remote', () => {uniqueKey: 'aaa',wasAlreadyPresent: true,wasAlreadyHandled: false,+ forefront: false,});});diff --git a/test/e2e/request-queue-with-concurrency/actor/.actor/actor.json b/test/e2e/request-queue-with-concurrency/actor/.actor/actor.jsonnew file mode 100644index 000000000..972ffaa33--- /dev/null+++ b/test/e2e/request-queue-with-concurrency/actor/.actor/actor.json@@ -0,0 +1,7 @@+{+ "actorSpecification": 1,+ "name": "test-request-queue-with-concurrency",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/request-queue-with-concurrency/actor/.gitignore b/test/e2e/request-queue-with-concurrency/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/request-queue-with-concurrency/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/request-queue-with-concurrency/actor/Dockerfile b/test/e2e/request-queue-with-concurrency/actor/Dockerfilenew file mode 100644index 000000000..36afd80b9--- /dev/null+++ b/test/e2e/request-queue-with-concurrency/actor/Dockerfile@@ -0,0 +1,16 @@+FROM apify/actor-node:20-beta++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional --no-audit \+ && npm update --no-audit \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/request-queue-with-concurrency/actor/main.js b/test/e2e/request-queue-with-concurrency/actor/main.jsnew file mode 100644index 000000000..023d0e8a9--- /dev/null+++ b/test/e2e/request-queue-with-concurrency/actor/main.js@@ -0,0 +1,52 @@+import { BasicCrawler } from '@crawlee/basic';+import { RequestQueue } from '@crawlee/core';+import { log } from '@crawlee/core';+import { Actor } from 'apify';+import { setTimeout } from 'node:timers/promises';++log.setLevel(log.LEVELS.DEBUG);++process.env.CRAWLEE_INTERNAL_TIMEOUT = '30000';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage:+ process.env.STORAGE_IMPLEMENTATION === 'LOCAL'+ ? new (await import('@apify/storage-local')).ApifyStorageLocal()+ : undefined,+};++await Actor.main(async () => {+ const input = await Actor.getInputOrThrow();++ log.info('Starting the crawler', input);++ const requestQueue = await RequestQueue.open(input.queueId);++ const crawler = new BasicCrawler({+ requestQueue,+ async requestHandler({ request, addRequests, pushData }) {+ await setTimeout(Math.random() * 200); // Simulate some work++ const url = new URL(request.url);+ const index = Number(url.pathname.slice(1));+ const newUrls = [];++ for (const newIndex of [index + 1, index + 2, 2 * index]) {+ if (newIndex <= input.limit) {+ url.pathname = `/${newIndex + 1}`;+ newUrls.push(url.toString());+ }+ }++ log.info(`Enqueueing ${newUrls.length} new urls`);+ if (newUrls.length > 0) {+ await addRequests(newUrls);+ }++ await pushData({ url });+ },+ });++ await crawler.run(['https://example.tld/1']);+}, mainOptions);diff --git a/test/e2e/request-queue-with-concurrency/actor/package.json b/test/e2e/request-queue-with-concurrency/actor/package.jsonnew file mode 100644index 000000000..381cdb7db--- /dev/null+++ b/test/e2e/request-queue-with-concurrency/actor/package.json@@ -0,0 +1,28 @@+{+ "name": "test-request-queue-with-concurrency",+ "version": "0.0.1",+ "description": "Request Queue Test - Zero Concurrency",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.3",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/request-queue-with-concurrency/test.mjs b/test/e2e/request-queue-with-concurrency/test.mjsnew file mode 100644index 000000000..6b1d48043--- /dev/null+++ b/test/e2e/request-queue-with-concurrency/test.mjs@@ -0,0 +1,45 @@+import { initialize, getActorTestDir, pushActor, startActorOnPlatform, expect } from '../tools.mjs';+import { Actor } from 'apify';+import { log } from 'crawlee';+import { setTimeout } from 'node:timers/promises';++if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {+ const testActorDirname = getActorTestDir(import.meta.url);+ await initialize(testActorDirname);++ const client = Actor.newClient();+ const actorId = await pushActor(client, testActorDirname);+ log.info(`Actor build (ID ${actorId})`);++ const { id: queueId } = await client.requestQueues().getOrCreate();+ const limit = 1000;++ const input = JSON.stringify({ limit, queueId });++ const runIds = await Promise.all([+ startActorOnPlatform(client, actorId, input),+ startActorOnPlatform(client, actorId, input),+ startActorOnPlatform(client, actorId, input),+ startActorOnPlatform(client, actorId, input),+ ]);+ log.info(`Started runs (IDs ${JSON.stringify(runIds)})`);++ const stats = await Promise.all(+ runIds.map(async (runId) => {+ await client.run(runId).waitForFinish();+ await setTimeout(6000);+ return client.run(runId).get();+ }),+ );+ log.info('Runs finished');++ const requestQueue = await client.requestQueue(queueId).get();++ await expect(requestQueue.pendingRequestCount === 0, 'No pending requests');+ await expect(requestQueue.handledRequestCount === 1000, 'All requests are handled');++ const firstActorFinish = Math.min(...stats.map((run) => run.finishedAt.valueOf()));+ const lastActorFinish = Math.max(...stats.map((run) => run.finishedAt.valueOf()));++ await expect(lastActorFinish - firstActorFinish < 60 * 1000, 'Actors finished at similar times');+}diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex f82bfb836..8f7875375 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -69,6 +69,79 @@ export function getActorTestDir(url) {return join(actorDirName, 'actor');}+export async function pushActor(client, dirName) {+ await copyPackages(dirName);+ try {+ execSync('npx -y apify-cli@beta push --no-prompt', { cwd: dirName });+ } catch (err) {+ console.error(colors.red(`Failed to push actor to the Apify platform. (signal ${colors.yellow(err.signal)})`));++ if (err.stdout) {+ console.log(colors.grey(` STDOUT: `), err.stdout);+ }++ if (err.stderr) {+ console.log(colors.red(` STDERR: `), err.stderr);+ }++ throw err;+ }++ const actorName = await getActorName(dirName);+ const { items: actors } = await client.actors().list();+ const { id } = actors.find((actor) => actor.name === actorName);++ return id;+}++export async function startActorOnPlatform(client, id, input, inputContentType = 'application/json', memory = 4096) {+ const gotClient = got.extend({+ retry: {+ limit: 2,+ statusCodes: [500, 502],+ },+ headers: {+ 'user-agent': 'crawlee e2e tests (got)',+ },+ timeout: {+ request: 10000,+ },+ });++ // Do NOT use Apify Client yet!+ // See https://github.com/apify/apify-client-js/issues/277++ try {+ const {+ data: { id: foundRunId },+ } = await gotClient(`https://api.apify.com/v2/acts/${id}/runs`, {+ method: 'POST',+ searchParams: {+ memory,+ },+ headers: {+ 'content-type': inputContentType,+ authorization: `Bearer ${client.token}`,+ },+ body: input,+ retry: {+ limit: 2,+ statusCodes: [500, 502],+ },+ }).json();++ return foundRunId;+ } catch (err) {+ console.error(colors.red(`Failed to start actor run on the Apify platform. (code ${colors.yellow(err.code)})`));++ if (err.response) {+ console.log(colors.grey(` RESPONSE: `), err.response.body || err.response.rawBody?.toString('utf-8'));+ }++ throw err;+ }+}+/*** @param {string} dirName* @param {number} [memory=4096]@@ -85,78 +158,8 @@ export async function runActor(dirName, memory = 4096) {if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {const client = Actor.newClient();-- await copyPackages(dirName);- try {- execSync('npx -y apify-cli@beta push --no-prompt', { cwd: dirName });- } catch (err) {- console.error(- colors.red(`Failed to push actor to the Apify platform. (signal ${colors.yellow(err.signal)})`),- );-- if (err.stdout) {- console.log(colors.grey(` STDOUT: `), err.stdout);- }-- if (err.stderr) {- console.log(colors.red(` STDERR: `), err.stderr);- }-- throw err;- }-- const actorName = await getActorName(dirName);- const { items: actors } = await client.actors().list();- const { id } = actors.find((actor) => actor.name === actorName);-- const gotClient = got.extend({- retry: {- limit: 2,- statusCodes: [500, 502],- },- headers: {- 'user-agent': 'crawlee e2e tests (got)',- },- timeout: {- request: 10000,- },- });-- // Do NOT use Apify Client yet!- // See https://github.com/apify/apify-client-js/issues/277- let runId;-- try {- const {- data: { id: foundRunId },- } = await gotClient(`https://api.apify.com/v2/acts/${id}/runs`, {- method: 'POST',- searchParams: {- memory,- },- headers: {- 'content-type': contentType,- authorization: `Bearer ${client.token}`,- },- body: input,- retry: {- limit: 2,- statusCodes: [500, 502],- },- }).json();-- runId = foundRunId;- } catch (err) {- console.error(- colors.red(`Failed to start actor run on the Apify platform. (code ${colors.yellow(err.code)})`),- );-- if (err.response) {- console.log(colors.grey(` RESPONSE: `), err.response.body || err.response.rawBody?.toString('utf-8'));- }-- throw err;- }+ const id = await pushActor(client, dirName);+ const runId = await startActorOnPlatform(client, id, input, contentType, memory);const {defaultKeyValueStoreId,