Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): stop draining async iterables after maxNewRequests is exhausted (#3579)
Junghwan5e834e96255a
diff --git a/docs/upgrading/upgrading-v4/request-loaders.md b/docs/upgrading/upgrading-v4/request-loaders.mdindex 86b969e52..47dcd5f6d 100644--- a/docs/upgrading/upgrading-v4/request-loaders.md+++ b/docs/upgrading/upgrading-v4/request-loaders.md@@ -16,6 +16,10 @@ The request loader/manager interfaces have been reworked. See the new [Request lRequests that the storage backend reports as unprocessed are now warned about and skipped after the first attempt, instead of being retried a bounded number of times. What a backend reports as unprocessed is a semantic rejection (typically malformed request data) that re-sending cannot fix — retrying transient failures is the storage backend's own responsibility.+### `maxNewRequests` no longer implies `waitForAllRequestsToBeAdded`++`RequestQueue.addRequestsBatched` used to wait for every batch (and drain the whole input) whenever `maxNewRequests` was set, so it could report `requestsOverLimit`. Draining could block forever on an async producer, so the input is now left unconsumed past the budget, and `requestsOverLimit` is only populated when you also pass `waitForAllRequestsToBeAdded: true` (otherwise it is `undefined`). `crawler.addRequests()` still reports skipped requests as before.+### `IRequestList` renamed to `IRequestLoader`The `IRequestList` interface has been renamed to `IRequestLoader` and is now the read-only base interface implemented by `RequestList` and `SitemapRequestLoader`. The writable `IRequestManager` interface now **extends** `IRequestLoader` with the request-adding and reclaiming surface (`addRequest`, `addRequestsBatched`, `reclaimRequest`, optional `purge`). There is no `IRequestList` alias — update your imports and type references to `IRequestLoader` (or `IRequestManager` if you need the write surface).diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 59d2e1828..61a400937 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -2221,7 +2221,7 @@ export class BasicCrawler<const result = await this.requestManager!.addRequestsBatched(filteredRequests(), {forefront: options.forefront,- waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,+ waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,batchSize: options.batchSize,waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,maxNewRequests: requestLimit,diff --git a/packages/core/src/storages/batched-adds.ts b/packages/core/src/storages/batched-adds.tsindex c3cac2320..ae451accd 100644--- a/packages/core/src/storages/batched-adds.ts+++ b/packages/core/src/storages/batched-adds.ts@@ -62,8 +62,11 @@ export async function drainRequestBatches<TItem extends Source>(const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';+ // Leftovers are only reported when the caller waits anyway: draining an async producer just to find+ // them could block indefinitely.+ const reportsOverLimit = maxNewRequests !== undefined && waitForAllRequestsToBeAdded;+let remainingBudget = maxNewRequests ?? Infinity;- const requestsOverLimit: Source[] = [];// Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;@@ -85,12 +88,15 @@ export async function drainRequestBatches<TItem extends Source>(addedRequests: ProcessedRequest[],waitForAll: Promise<ProcessedRequest[]>,): Promise<AddRequestsBatchedResult> => {- if (maxNewRequests !== undefined) {- // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever- // is left is still sitting in `items` rather than in a chunk we have seen.- for await (const item of items) {- requestsOverLimit.push(item);- }+ if (!reportsOverLimit) {+ return { addedRequests, waitForAllRequestsToBeAdded: waitForAll };+ }++ // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever+ // is left is still sitting in `items` rather than in a chunk we have seen.+ const requestsOverLimit: Source[] = [];+ for await (const item of items) {+ requestsOverLimit.push(item);}return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };@@ -118,9 +124,8 @@ export async function drainRequestBatches<TItem extends Source>(return added;};- // With a budget we must drain everything before we can report what went over it; under `deferred` a- // writer that finishes after commit would have nowhere to put its journal entries.- const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;+ // Under `deferred` a writer that finishes after commit would have nowhere to put its journal entries.+ const awaitsRemainder = waitForAllRequestsToBeAdded || deferred;// An un-awaited writer outlives the transaction scope it inherits, so it must not record into a// transaction that may already be closed. It writes directly - its write-through additions were neverdiff --git a/packages/core/src/storages/request-queue.ts b/packages/core/src/storages/request-queue.tsindex d077c5c3b..52cd50d99 100644--- a/packages/core/src/storages/request-queue.ts+++ b/packages/core/src/storages/request-queue.ts@@ -1166,13 +1166,11 @@ export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions/*** If set, only this many *actually new* requests (i.e. not already present in the queue) will be added.- * Once the budget is reached, remaining requests from the iterable will be collected in- * {@apilink AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} instead.+ * Once the budget is reached, remaining requests from the iterable are left unconsumed, or collected in+ * {@apilink AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} if+ * {@apilink AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} is set.** This is useful in combination with `maxRequestsPerCrawl` to avoid duplicate URLs consuming the budget.- *- * **Note:** Setting this option implicitly enables {@apilink AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`},- * since all batches must complete before leftover requests can be accurately reported.*/maxNewRequests?: number;}@@ -1200,7 +1198,8 @@ export interface AddRequestsBatchedResult {/*** Requests from the input that were not added to the queue because the* {@apilink AddRequestsBatchedOptions.maxNewRequests|`maxNewRequests`} budget was reached.- * Empty when `maxNewRequests` is not set.+ * `undefined` unless both `maxNewRequests` and+ * {@apilink AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} are set.*/requestsOverLimit?: Source[];}diff --git a/test/basic-crawler/throttling-request-manager.test.ts b/test/basic-crawler/throttling-request-manager.test.tsindex dd2d0643c..eb292cb03 100644--- a/test/basic-crawler/throttling-request-manager.test.ts+++ b/test/basic-crawler/throttling-request-manager.test.ts@@ -142,7 +142,11 @@ describe('ThrottlingRequestManager', () => {}}- const result = await manager.addRequestsBatched(urls(), { batchSize: 2, maxNewRequests: 4 });+ const result = await manager.addRequestsBatched(urls(), {+ batchSize: 2,+ maxNewRequests: 4,+ waitForAllRequestsToBeAdded: true,+ });expect(await manager.getTotalCount()).toBe(4);expect(result.requestsOverLimit).toHaveLength(6);diff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.tsindex 6fa722bd1..9e38a29b4 100644--- a/test/core/crawlers/basic-crawler.test.ts+++ b/test/core/crawlers/basic-crawler.test.ts@@ -3983,7 +3983,7 @@ describe('BasicCrawler', () => {{ url: 'http://example.com/d' },{ url: 'http://example.com/e' },],- { maxNewRequests: 2 },+ { maxNewRequests: 2, waitForAllRequestsToBeAdded: true },);const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);@@ -4005,7 +4005,10 @@ describe('BasicCrawler', () => {yield { url: 'http://example.com/e' };}- const result = await queue.addRequestsBatched(urls(), { maxNewRequests: 2 });+ const result = await queue.addRequestsBatched(urls(), {+ maxNewRequests: 2,+ waitForAllRequestsToBeAdded: true,+ });const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);@@ -4015,6 +4018,65 @@ describe('BasicCrawler', () => {expect(overLimitUrls).toHaveLength(3);});+ test('addRequestsBatched with maxNewRequests should not wait for an async iterable beyond the remaining budget', async () => {+ const queue = await RequestQueue.open();+ let consumed = 0;+ let releaseBlockedRequest = () => {};+ const blockedRequest = new Promise<void>((resolve) => {+ releaseBlockedRequest = resolve;+ });++ async function* urls() {+ consumed += 1;+ yield { url: 'http://example.com/a' };++ consumed += 1;+ await blockedRequest;+ yield { url: 'http://example.com/b' };+ }++ const pendingResult = queue.addRequestsBatched(urls(), {+ maxNewRequests: 1,+ waitBetweenBatchesMillis: 0,+ });++ const raced = await Promise.race([+ pendingResult.then(() => 'resolved'),+ new Promise<'timeout'>((resolve) => setTimeout(() => resolve('timeout'), 100)),+ ]);++ expect(raced).toBe('resolved');+ expect(consumed).toBe(1);++ releaseBlockedRequest();+ await pendingResult;+ });++ test('addRequests should still report skipped requests for maxRequestsPerCrawl when the source is async', async () => {+ const requestQueue = await RequestQueue.open();+ const onSkippedRequestMock = vitest.fn();++ const crawler = new BasicCrawler({+ requestQueue,+ maxRequestsPerCrawl: 2,+ onSkippedRequest: onSkippedRequestMock,+ requestHandler: async () => {},+ });++ async function* urls() {+ yield 'http://example.com/a';+ yield 'http://example.com/b';+ yield 'http://example.com/c';+ }++ await crawler.addRequests(urls());++ expect(onSkippedRequestMock).toHaveBeenCalledWith({+ request: expect.objectContaining({ url: 'http://example.com/c' }),+ reason: 'limit',+ });+ });+test('should not count duplicate URLs toward maxRequestsPerCrawl limit (enqueueLinks)', async () => {const requestQueue = await RequestQueue.open();