Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): stop scanning handled requests in the in-memory request queue (#4083)
Vlad Frangufde8d84c46a2
diff --git a/packages/core/src/memory-storage/resource-clients/request-queue.ts b/packages/core/src/memory-storage/resource-clients/request-queue.tsindex 5290d21b6..6dbd04218 100644--- a/packages/core/src/memory-storage/resource-clients/request-queue.ts+++ b/packages/core/src/memory-storage/resource-clients/request-queue.ts@@ -46,11 +46,11 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQupendingRequestCount = 0;/*** Serializes every operation that reads-then-writes this backend's shared queue state — the- * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request- * counts. Those mutations span `await` points, so without this mutex a concurrent operation could- * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while- * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/- * `isFinished`, whose head scan also prunes `forefrontRequestIds`.+ * `requests` map, the `pendingRequestIds` set, the `forefrontRequestIds` array, the+ * `inProgressRequestIds` set and the request counts. Those mutations span `await` points, so+ * without this mutex a concurrent operation could interleave and corrupt them (e.g. a head scan+ * pruning `forefrontRequestIds` while `addBatchOfRequests` pushes to it). Held by every mutating+ * method as well as by `isEmpty`/`isFinished`, whose head scan also prunes `forefrontRequestIds`.*/readonly #queueStateMutex = new AsyncQueue();#forefrontRequestIds: string[] = [];@@ -66,6 +66,13 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQureadonly #inProgressRequestIds = new Set<string>();readonly #requests = new Map<string, InternalRequest>();++ /**+ * IDs of requests that are not yet handled (pending or in progress), in insertion order. Handled+ * requests stay in `requests` for deduplication but are removed from here, so head scans only+ * ever walk the unhandled tail instead of every request the queue has ever seen.+ */+ readonly #pendingRequestIds = new Set<string>();// kept as TS-private: storage-backend tests read this field at runtimeprivate readonly storageBackend: MemoryStorageBackend;@@ -94,6 +101,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head// scan would resolve to a missing request and dereference.this.#requests.clear();+ this.#pendingRequestIds.clear();this.#forefrontRequestIds = [];this.#inProgressRequestIds.clear();}@@ -110,6 +118,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQutry {// Clear all in-memory statethis.#requests.clear();+ this.#pendingRequestIds.clear();this.#forefrontRequestIds = [];this.#inProgressRequestIds.clear();this.handledRequestCount = 0;@@ -126,9 +135,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuyield this.#forefrontRequestIds[i];}- for (const key of this.#requests.keys()) {- yield key;- }+ yield* this.#pendingRequestIds;}/**@@ -144,7 +151,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu* Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it* forces a scan of every pending entry even when only `limit` items are wanted. Callers that only* need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as- * soon as the page is filled, keeping those calls O(head) instead of O(N).+ * soon as the page is filled, keeping those calls O(head) instead of O(pending).*/private async listPendingHead(limit: number,@@ -173,11 +180,10 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuconst request = this.#requests.get(requestId)!;- // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.+ // Only `forefrontRequestIds` can still reference a handled request (`orderNo === null`);+ // `pendingRequestIds` drops them on handling. Remember the id so the list gets pruned below.if (request.orderNo === null) {- if (this.#forefrontRequestIds.includes(requestId)) {- handledForefrontIds.add(requestId);- }+ handledForefrontIds.add(requestId);continue;}@@ -262,6 +268,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuthis.#requests.set(requestModel.id, requestModel);if (requestModel.orderNo) {+ this.#pendingRequestIds.add(requestModel.id);this.pendingRequestCount += 1;} else {this.handledRequestCount += 1;@@ -325,6 +332,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuconst requestModel = this.createInternalRequest({ ...request, handledAt }, false);this.#requests.set(id, requestModel);+ this.#pendingRequestIds.delete(id);// The request is no longer in progress for this client.this.#inProgressRequestIds.delete(id);diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex ec0f82db1..6d8a336d7 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -703,3 +703,39 @@ describe('RequestQueue background batches', () => {expect((await queue.checkReadiness()).status).toBe('finished');}, 10_000);});++describe('MemoryStorageBackend request queue', () => {+ test('head operations do not scan already-handled requests', async () => {+ const backend = new MemoryStorageBackend();+ const queue = await backend.createRequestQueueBackend({ name: 'handled-scan' });++ // Simulate a crawl nearing its end: a large number of handled requests and few pending ones.+ const handledAt = new Date().toISOString();+ const handledCount = 200_000;+ for (let i = 0; i < handledCount; i += 1_000) {+ await queue.addBatchOfRequests(+ Array.from({ length: 1_000 }, (_, j) => ({+ url: `http://example.com/${i + j}`,+ uniqueKey: `handled-${i + j}`,+ handledAt,+ })),+ );+ }+ expect((await queue.getMetadata()).handledRequestCount).toBe(handledCount);++ // Each call used to walk the whole map (O(handled)): ~8s for this loop vs ~5ms once only pending+ // requests are scanned. The loose budget only separates those two regimes, well above CI noise.+ const start = performance.now();+ for (let i = 0; i < 200; i++) {+ await queue.addBatchOfRequests([{ url: `http://example.com/new-${i}`, uniqueKey: `new-${i}` }]);+ expect(await queue.isEmpty()).toBe(false);+ const request = await queue.fetchNextRequest();+ expect(request!.uniqueKey).toBe(`new-${i}`);+ expect(await queue.isFinished()).toBe(false);+ await queue.markRequestAsHandled({ ...request!, handledAt });+ }+ expect(await queue.isEmpty()).toBe(true);+ expect(await queue.isFinished()).toBe(true);+ expect(performance.now() - start).toBeLessThan(500);+ }, 60_000);+});