Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(RequestQueueV2): remove `inProgress` cache, rely solely on locked states (#2601)
Vlad Frangu57fcb0804a9f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 5d6f12c25..f10a498dd 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -18,7 +18,6 @@ import type {ProxyInfo,Request,RequestOptions,- RequestProvider,RouterHandler,RouterRoutes,Session,@@ -40,6 +39,7 @@ import {mergeCookies,NonRetryableError,purgeDefaultStorages,+ RequestProvider,RequestQueueV1,RequestQueue,RequestState,@@ -1185,16 +1185,23 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn false;}- // eslint-disable-next-line dot-notation- source['inProgress'].delete(request.id!);+ if (source instanceof RequestQueueV1) {+ // eslint-disable-next-line dot-notation+ source['inProgress']?.delete(request.id!);+ }+const delay = lastAccessTime + this.sameDomainDelayMillis - now;this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`,);setTimeout(async () => {this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);- // eslint-disable-next-line dot-notation- source['inProgress'].add(request.id!);++ if (source instanceof RequestQueueV1) {+ // eslint-disable-next-line dot-notation+ source['inProgress'].add(request.id!);+ }+await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });}, delay);@@ -1350,6 +1357,15 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextawait this._cleanupContext(crawlingContext);this.crawlingContexts.delete(crawlingContext.id);++ if (source instanceof RequestProvider) {+ // Always release a lock on a request at the end of the cycle+ try {+ await source.client.deleteRequestLock(request.id!);+ } catch {+ // We don't have the lock, or the request was never locked. Either way it's fine+ }+ }}}diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.tsindex 8d8539e0d..45f1466d4 100644--- a/packages/core/src/storages/request_provider.ts+++ b/packages/core/src/storages/request_provider.ts@@ -48,14 +48,15 @@ export abstract class RequestProvider implements IStorage {protected queueHeadIds = new ListDictionary<string>();protected requestCache: LruCache<RequestLruItem>;- /** @internal */- inProgress = new Set<string>();+protected recentlyHandledRequestsCache: LruCache<boolean>;protected queuePausedForMigration = false;protected lastActivity = new Date();+ protected isFinishedCalledWhileHeadWasNotEmpty = 0;+constructor(options: InternalRequestProviderOptions,readonly config = Configuration.getGlobalConfig(),@@ -71,7 +72,7 @@ export abstract class RequestProvider implements IStorage {this.requestCache = new LruCache({ maxLength: options.requestCacheMaxSize });this.recentlyHandledRequestsCache = new LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });- this.log = log.child({ prefix: options.logPrefix });+ this.log = log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });const eventManager = config.getEventManager();@@ -80,13 +81,6 @@ export abstract class RequestProvider implements IStorage {});}- /**- * @ignore- */- inProgressCount() {- return this.inProgress.size;- }-/*** Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).*@@ -166,11 +160,7 @@ export abstract class RequestProvider implements IStorage {const { requestId, wasAlreadyPresent } = queueOperationInfo;this._cacheRequest(cacheKey, queueOperationInfo);- if (- !wasAlreadyPresent &&- !this.inProgress.has(requestId) &&- !this.recentlyHandledRequestsCache.get(requestId)- ) {+ if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {this.assumedTotalCount++;// Performance optimization: add request straight to head if possible@@ -287,11 +277,7 @@ export abstract class RequestProvider implements IStorage {this._cacheRequest(cacheKey, newRequest);}- if (- !wasAlreadyPresent &&- !this.inProgress.has(requestId) &&- !this.recentlyHandledRequestsCache.get(requestId)- ) {+ if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {this.assumedTotalCount++;// Performance optimization: add request straight to head if possible@@ -486,13 +472,6 @@ export abstract class RequestProvider implements IStorage {}),);- if (!this.inProgress.has(request.id)) {- this.log.debug(`Cannot mark request ${request.id} as handled, because it is not in progress!`, {- requestId: request.id,- });- return null;- }-const handledAt = request.handledAt ?? new Date().toISOString();const queueOperationInfo = (await this.client.updateRequest({...request,@@ -501,13 +480,14 @@ export abstract class RequestProvider implements IStorage {request.handledAt = handledAt;queueOperationInfo.uniqueKey = request.uniqueKey;- this.inProgress.delete(request.id);this.recentlyHandledRequestsCache.add(request.id, true);if (!queueOperationInfo.wasAlreadyHandled) {this.assumedHandledCount++;}+ this.queueHeadIds.remove(request.id);+this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);return queueOperationInfo;@@ -543,13 +523,6 @@ export abstract class RequestProvider implements IStorage {const { forefront = false } = options;- if (!this.inProgress.has(request.id)) {- this.log.debug(`Cannot reclaim request ${request.id}, because it is not in progress!`, {- requestId: request.id,- });- return null;- }-// TODO: If request hasn't been changed since the last getRequest(),// we don't need to call updateRequest() and thus improve performance.const queueOperationInfo = (await this.client.updateRequest(request, {@@ -582,19 +555,22 @@ export abstract class RequestProvider implements IStorage {*/async isFinished(): Promise<boolean> {// TODO: once/if we figure out why sometimes request queues get stuck (if it's even request queues), remove this once and for all :)- if (Date.now() - +this.lastActivity > this.internalTimeoutMillis) {- const message = `The request queue seems to be stuck for ${+ if (Date.now() - this.lastActivity.getTime() > this.internalTimeoutMillis) {+ const maybeHead = await this.client.listHead({ limit: 1 });++ const request = maybeHead.items[0] ? await this.client.getRequest(maybeHead.items[0].id) : null;++ const message = `The request queue hasn't had activity for ${this.internalTimeoutMillis / 1000}s, resetting internal state.`;this.log.warning(message, {- inProgress: [...this.inProgress],queueHeadIdsPending: this.queueHeadIds.length(),+ hasPendingOrLockedRequests: maybeHead.items.length > 0,+ pendingRequestIsLocked: request?.lockExpiresAt,});- // We only need to reset these two variables, no need to reset all the other statsthis.queueHeadIds.clear();- this.inProgress.clear();}if (this.queueHeadIds.length() > 0) {@@ -605,29 +581,53 @@ export abstract class RequestProvider implements IStorage {return false;}- if (this.inProgressCount() > 0) {- this.log.debug('There are still requests in progress (or zombie)', {- inProgress: [...this.inProgress],- });-- return false;- }-const currentHead = await this.client.listHead({ limit: 2 });if (currentHead.items.length !== 0) {- this.log.debug(- 'Queue head still returned requests that need to be processed (or that are locked by other clients)',- );+ // Give users some more concrete info as to why their crawlers seem to be "hanging" doing nothing while we're waiting because the queue is technically+ // not empty. We decided that a queue with elements in its head but that are also locked shouldn't return true in this function.+ // If that ever changes, this function might need a rewrite+ // The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much. This is also a very specific path that most crawlers shouldn't hit+ if (++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {+ const requests = await Promise.all(+ currentHead.items.map(async (item) => this.client.getRequest(item.id)),+ );++ this.log.info(+ `Queue head still returned requests that need to be processed (or that are locked by other clients)`,+ {+ requests: requests+ .map((r) => {+ if (!r) {+ return null;+ }++ return {+ id: r.id,+ lockExpiresAt: r.lockExpiresAt,+ lockedBy: r.lockByClient,+ };+ })+ .filter(Boolean),+ clientKey: this.clientKey,+ },+ );+ } else {+ this.log.debug(+ 'Queue head still returned requests that need to be processed (or that are locked by other clients)',+ {+ requestIds: currentHead.items.map((item) => item.id),+ },+ );+ }}- return currentHead.items.length === 0 && this.inProgressCount() === 0;+ return currentHead.items.length === 0;}protected _reset() {this.lastActivity = new Date();this.queueHeadIds.clear();- this.inProgress.clear();this.recentlyHandledRequestsCache.clear();this.assumedTotalCount = 0;this.assumedHandledCount = 0;diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 6331f2954..24b3e6cbd 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -4,7 +4,7 @@ import { REQUEST_QUEUE_HEAD_MAX_LIMIT } from '@apify/consts';import type { Dictionary } from '@crawlee/types';import { checkStorageAccess } from './access_checking';-import type { RequestProviderOptions } from './request_provider';+import type { RequestProviderOptions, RequestQueueOperationInfo } from './request_provider';import { RequestProvider } from './request_provider';import {API_PROCESSED_REQUESTS_DELAY_MILLIS,@@ -83,6 +83,8 @@ class RequestQueue extends RequestProvider {hadMultipleClients?: boolean;}> | null = null;+ private inProgress = new Set<string>();+/*** @internal*/@@ -98,6 +100,13 @@ class RequestQueue extends RequestProvider {);}+ /**+ * @internal+ */+ public inProgressCount(): number {+ return this.inProgress.size;+ }+/*** Returns a next request in the queue to be processed, or `null` if there are no more pending requests.*@@ -346,6 +355,23 @@ class RequestQueue extends RequestProvider {return result;}+ /**+ * @inheritdoc+ */+ override async markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null> {+ const res = await super.markRequestHandled(request);++ this.inProgress.delete(request.id!);++ return res;+ }++ protected override _reset(): void {+ super._reset();++ this.inProgress.clear();+ }+/*** Opens a request queue and returns a promise resolving to an instance* of the {@apilink RequestQueue} class.diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.tsindex 4f4f12484..41dfc5e71 100644--- a/packages/core/src/storages/request_queue_v2.ts+++ b/packages/core/src/storages/request_queue_v2.ts@@ -3,7 +3,7 @@ import type { Dictionary } from '@crawlee/types';import { checkStorageAccess } from './access_checking';import type { RequestQueueOperationInfo, RequestProviderOptions } from './request_provider';import { RequestProvider } from './request_provider';-import { STORAGE_CONSISTENCY_DELAY_MILLIS, getRequestId } from './utils';+import { getRequestId } from './utils';import { Configuration } from '../configuration';import { EventType } from '../events';import type { Request } from '../request';@@ -112,26 +112,15 @@ export class RequestQueue extends RequestProvider {}// This should never happen, but...- if (this.inProgress.has(nextRequestId) || this.recentlyHandledRequestsCache.get(nextRequestId)) {- this.log.warning('Queue head returned a request that is already in progress?!', {+ if (this.recentlyHandledRequestsCache.get(nextRequestId)) {+ this.log.warning('Queue head returned a request that was recently marked as handled?!', {nextRequestId,- inProgress: this.inProgress.has(nextRequestId),recentlyHandled: !!this.recentlyHandledRequestsCache.get(nextRequestId),});return null;}- this.inProgress.add(nextRequestId);-- let request: Request | null;-- try {- request = await this.getOrHydrateRequest(nextRequestId);- } catch (e) {- // On error, remove the request from in progress, otherwise it would be there forever- this.inProgress.delete(nextRequestId);- throw e;- }+ const request: Request | null = await this.getOrHydrateRequest(nextRequestId);// NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:@@ -145,10 +134,6 @@ export class RequestQueue extends RequestProvider {nextRequestId,});- setTimeout(() => {- this.inProgress.delete(nextRequestId);- }, STORAGE_CONSISTENCY_DELAY_MILLIS);-return null;}@@ -176,11 +161,6 @@ export class RequestQueue extends RequestProvider {if (res) {const [request, options] = args;- // Mark the request as no longer in progress,- // as the moment we delete the lock, we could end up also re-fetching the request in a subsequent ensureHeadIsNonEmpty()- // which could potentially lock the request again- this.inProgress.delete(request.id!);-// Try to delete the request lock if possibletry {await this.client.deleteRequestLock(request.id!, { forefront: options?.forefront ?? false });@@ -217,11 +197,10 @@ export class RequestQueue extends RequestProvider {for (const { id, uniqueKey } of headData.items) {// Queue head index might be behind the main table, so ensure we don't recycle requests- if (!id || !uniqueKey || this.inProgress.has(id) || this.recentlyHandledRequestsCache.get(id)) {- this.log.debug(`Skipping request from queue head, already in progress or recently handled`, {+ if (!id || !uniqueKey || this.recentlyHandledRequestsCache.get(id)) {+ this.log.debug(`Skipping request from queue head as it's invalid or recently handled`, {id,uniqueKey,- inProgress: this.inProgress.has(id),recentlyHandled: !!this.recentlyHandledRequestsCache.get(id),});diff --git a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.tsindex 086c22dcc..d5c17388d 100644--- a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts+++ b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts@@ -27,7 +27,11 @@ describe('RequestQueue#addRequest should not call the API if the request is alreexpect(clientSpy).toHaveBeenCalledTimes(1);- await requestQueue.markRequestHandled({ id: requestData.requestId, uniqueKey: requestData.uniqueKey } as any);+ await requestQueue.markRequestHandled({+ id: requestData.requestId,+ url: 'https://example.com',+ uniqueKey: requestData.uniqueKey,+ } as any);await requestQueue.addRequest({ url: 'https://example.com' });@@ -48,6 +52,7 @@ describe('RequestQueue#addRequests should not call the API if the request is alrawait requestQueue.markRequestHandled({id: requestData.processedRequests[0].requestId,uniqueKey: requestData.processedRequests[0].uniqueKey,+ url: 'https://example2.com',} as any);await requestQueue.addRequests([{ url: 'https://example2.com' }]);diff --git a/packages/core/test/request-queue/request-queue-v2.test.ts b/packages/core/test/request-queue/request-queue-v2.test.tsindex a9c85ddd4..f311ac94d 100644--- a/packages/core/test/request-queue/request-queue-v2.test.ts+++ b/packages/core/test/request-queue/request-queue-v2.test.ts@@ -1,15 +1,7 @@-/* eslint-disable dot-notation */import { MemoryStorage } from '@crawlee/memory-storage';-import type {- ListAndLockHeadResult,- ListAndLockOptions,- ListOptions,- ProlongRequestLockOptions,- ProlongRequestLockResult,- QueueHead,-} from '@crawlee/types';+import type { ListAndLockHeadResult } from '@crawlee/types';import { RequestQueueV2 } from 'crawlee';-import type { SpyInstance } from 'vitest';+import type { MockInstance } from 'vitest';const storage = new MemoryStorage({ persistStorage: false, writeMetadata: false });@@ -31,9 +23,9 @@ vitest.setConfig({ restoreMocks: false });describe('RequestQueueV2#isFinished should use listHead instead of listAndLock', () => {let queue: RequestQueueV2;- let clientListHeadSpy: SpyInstance<[options?: ListOptions | undefined], Promise<QueueHead>>;+ let clientListHeadSpy: MockInstance<typeof queue.client.listHead>;let listHeadCallCount = 0;- let clientListAndLockHeadSpy: SpyInstance<[options: ListAndLockOptions], Promise<ListAndLockHeadResult>>;+ let clientListAndLockHeadSpy: MockInstance<typeof queue.client.listAndLockHead>;let listAndLockHeadCallCount = 0;let lockResult: ListAndLockHeadResult;@@ -62,9 +54,9 @@ describe('RequestQueueV2#isFinished should use listHead instead of listAndLock',describe('RequestQueueV2#isFinished should return true once locked requests are handled', () => {let queue: RequestQueueV2;- let clientListHeadSpy: SpyInstance<[options?: ListOptions | undefined], Promise<QueueHead>>;+ let clientListHeadSpy: MockInstance<typeof queue.client.listHead>;let listHeadCallCount = 0;- let clientListAndLockHeadSpy: SpyInstance<[options: ListAndLockOptions], Promise<ListAndLockHeadResult>>;+ let clientListAndLockHeadSpy: MockInstance<typeof queue.client.listAndLockHead>;let lockResult: ListAndLockHeadResult;beforeAll(async () => {@@ -73,7 +65,8 @@ describe('RequestQueueV2#isFinished should return true once locked requests areclientListAndLockHeadSpy = vitest.spyOn(queue.client, 'listAndLockHead');lockResult = await queue.client.listAndLockHead({ lockSecs: 60 });- queue['inProgress'].add(lockResult.items[0].id);+ // eslint-disable-next-line dot-notation+ queue['queueHeadIds'].add(lockResult.items[0].id, lockResult.items[0].id);});test('should return true once locked requests are handled', async () => {@@ -94,12 +87,9 @@ describe('RequestQueueV2#isFinished should return true once locked requests aredescribe('RequestQueueV2#fetchNextRequest should use locking API', () => {let queue: RequestQueueV2;- let clientListHeadSpy: SpyInstance<[options?: ListOptions | undefined], Promise<QueueHead>>;- let clientListAndLockHeadSpy: SpyInstance<[options: ListAndLockOptions], Promise<ListAndLockHeadResult>>;- let clientProlongLockSpy: SpyInstance<- [id: string, options: ProlongRequestLockOptions],- Promise<ProlongRequestLockResult>- >;+ let clientListHeadSpy: MockInstance<typeof queue.client.listHead>;+ let clientListAndLockHeadSpy: MockInstance<typeof queue.client.listAndLockHead>;+ let clientProlongLockSpy: MockInstance<typeof queue.client.prolongRequestLock>;let listAndLockHeadCallCount = 0;beforeAll(async () => {@@ -135,7 +125,6 @@ describe('RequestQueueV2#isEmpty should return true even if isFinished returns fbeforeAll(async () => {queue = await makeQueue('is-empty-vs-is-finished', 1);lockResult = await queue.client.listAndLockHead({ lockSecs: 60 });- queue['inProgress'].add(lockResult.items[0].id);});test('should return true when isFinished returns false', async () => {diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.tsindex 110395667..8f09c80e7 100644--- a/packages/memory-storage/src/resource-clients/request-queue.ts+++ b/packages/memory-storage/src/resource-clients/request-queue.ts@@ -308,7 +308,8 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueconst start = Date.now();- const isLocked = (r: InternalRequest) => !r.orderNo || r.orderNo > start || r.orderNo < -start;+ // If there is no `orderNo` -> request was marked as handled+ const isLocked = (r: InternalRequest) => r.orderNo && (r.orderNo > start || r.orderNo < -start);if (!isLocked(internalRequest)) {throw new Error(`Request with ID ${id} is not locked in queue ${queue.name ?? queue.id}`);}diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex aab8dc100..cb1d2d5c0 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -88,14 +88,6 @@ describe('RequestQueue remote', () => {expect(queue.inProgressCount()).toBe(1);// Test validations- await queue- .markRequestHandled(new Request({ id: 'XXX', url: 'https://example.com' }))- .catch((err) =>- expect(err.message).toMatch(/Cannot mark request XXX as handled, because it is not in progress/),- );- await queue- .reclaimRequest(new Request({ id: 'XXX', url: 'https://example.com' }))- .catch((err) => expect(err.message).toMatch(/Cannot reclaim request XXX, because it is not in progress/));await queue.addRequest(new Request({ id: 'id-already-set', url: 'https://example.com' })).catch((err) =>diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/.actor/actor.json b/test/e2e/request-queue-v2-zero-concurrency/actor/.actor/actor.jsondeleted file mode 100644index 959ec7eaa..000000000--- a/test/e2e/request-queue-v2-zero-concurrency/actor/.actor/actor.json+++ /dev/null@@ -1,7 +0,0 @@-{- "actorSpecification": 1,- "name": "test-request-queue-zero-concurrency",- "version": "0.0",- "buildTag": "latest",- "env": null-}diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/.gitignore b/test/e2e/request-queue-v2-zero-concurrency/actor/.gitignoredeleted file mode 100644index ced7cbfc5..000000000--- a/test/e2e/request-queue-v2-zero-concurrency/actor/.gitignore+++ /dev/null@@ -1,7 +0,0 @@-.idea-.DS_Store-node_modules-package-lock.json-apify_storage-crawlee_storage-storagediff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-v2-zero-concurrency/actor/Dockerfiledeleted file mode 100644index 36afd80b9..000000000--- a/test/e2e/request-queue-v2-zero-concurrency/actor/Dockerfile+++ /dev/null@@ -1,16 +0,0 @@-FROM apify/actor-node:20-beta--COPY packages ./packages-COPY package*.json ./--RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional --no-audit \- && npm update --no-audit \- && echo "Installed NPM packages:" \- && (npm list --only=prod --no-optional --all || true) \- && echo "Node.js version:" \- && node --version \- && echo "NPM version:" \- && npm --version--COPY . ./diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/main.js b/test/e2e/request-queue-v2-zero-concurrency/actor/main.jsdeleted file mode 100644index 2cb940fa8..000000000--- a/test/e2e/request-queue-v2-zero-concurrency/actor/main.js+++ /dev/null@@ -1,33 +0,0 @@-import { CheerioCrawler, log, RequestQueue } from '@crawlee/cheerio';-import { Actor } from 'apify';--log.setLevel(log.LEVELS.DEBUG);--process.env.CRAWLEE_INTERNAL_TIMEOUT = '30000';--const mainOptions = {- exit: Actor.isAtHome(),- storage:- process.env.STORAGE_IMPLEMENTATION === 'LOCAL'- ? new (await import('@apify/storage-local')).ApifyStorageLocal()- : undefined,-};--// RequestQueue auto-reset when stuck with requests in progress-await Actor.main(async () => {- const requestQueue = await RequestQueue.open();- await requestQueue.addRequest({ url: 'https://example.com/?q=1' });- await requestQueue.addRequest({ url: 'https://example.com/?q=2' });- const r3 = await requestQueue.addRequest({ url: 'https://example.com/?q=3' });- // trigger 0 concurrency by marking one of the requests as already in progress- requestQueue.inProgress.add(r3.requestId);-- const crawler = new CheerioCrawler({- requestQueue,- async requestHandler({ request }) {- log.info(request.id);- },- });-- await crawler.run();-}, mainOptions);diff --git a/test/e2e/request-queue-v2-zero-concurrency/actor/package.json b/test/e2e/request-queue-v2-zero-concurrency/actor/package.jsondeleted file mode 100644index 1f24f5ba2..000000000--- a/test/e2e/request-queue-v2-zero-concurrency/actor/package.json+++ /dev/null@@ -1,28 +0,0 @@-{- "name": "test-request-queue-zero-concurrency",- "version": "0.0.1",- "description": "Request Queue Test - Zero Concurrency",- "dependencies": {- "apify": "next",- "@apify/storage-local": "^2.1.3",- "@crawlee/basic": "file:./packages/basic-crawler",- "@crawlee/browser-pool": "file:./packages/browser-pool",- "@crawlee/http": "file:./packages/http-crawler",- "@crawlee/cheerio": "file:./packages/cheerio-crawler",- "@crawlee/core": "file:./packages/core",- "@crawlee/memory-storage": "file:./packages/memory-storage",- "@crawlee/types": "file:./packages/types",- "@crawlee/utils": "file:./packages/utils"- },- "overrides": {- "apify": {- "@crawlee/core": "file:./packages/core",- "@crawlee/utils": "file:./packages/utils"- }- },- "scripts": {- "start": "node main.js"- },- "type": "module",- "license": "ISC"-}diff --git a/test/e2e/request-queue-v2-zero-concurrency/test.mjs b/test/e2e/request-queue-v2-zero-concurrency/test.mjsdeleted file mode 100644index 42656d0ad..000000000--- a/test/e2e/request-queue-v2-zero-concurrency/test.mjs+++ /dev/null@@ -1,12 +0,0 @@-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';--const testActorDirname = getActorTestDir(import.meta.url);-await initialize(testActorDirname);--const { stats } = await runActor(testActorDirname);--await expect(stats.requestsFinished === 3, 'All requests finished');-await expect(- stats.crawlerRuntimeMillis > 30 * 1e3 && stats.crawlerRuntimeMillis < 35 * 1e3,- 'RequestQueue triggers auto-reset after being stuck with requests in progress',-);