Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: Accept (Async)Iterables in `addRequests` methods (#3013)

Jan Buchara4ab74852c3c
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index f0058d5d0..b531ef0a6 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -15,14 +15,13 @@ import type {
LoadedContext,
ProxyInfo,
Request,
- RequestOptions,
+ RequestsLike,
RestrictedCrawlingContext,
RouterHandler,
RouterRoutes,
Session,
SessionPoolOptions,
SkippedRequestCallback,
- Source,
StatisticsOptions,
StatisticState,
} from '@crawlee/core';
@@ -903,7 +902,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* @param [requests] The requests to add.
* @param [options] Options for the request queue.
*/
- async run(requests?: (string | Request | RequestOptions)[], options?: CrawlerRunOptions): Promise<FinalStatistics> {
+ async run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics> {
if (this.running) {
throw new Error(
'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.',
@@ -1100,33 +1099,40 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* @param options Options for the request queue
*/
async addRequests(
- requests: (string | Source)[],
+ requests: RequestsLike,
options: CrawlerAddRequestsOptions = {},
): Promise<CrawlerAddRequestsResult> {
const requestQueue = await this.getRequestQueue();
const requestLimit = this.calculateEnqueuedRequestLimit();
- const allowedRequests: (string | Source)[] = [];
const skippedBecauseOfRobots = new Set<string>();
const skippedBecauseOfLimit = new Set<string>();
- for (const request of requests) {
- const url = typeof request === 'string' ? request : request.url!;
+ const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
- if (requestLimit !== undefined && allowedRequests.length >= requestLimit) {
- skippedBecauseOfLimit.add(url);
- continue;
- }
+ async function* filteredRequests() {
+ let yieldedRequestCount = 0;
- if (await this.isAllowedBasedOnRobotsTxtFile(url)) {
- allowedRequests.push(request);
- } else {
- skippedBecauseOfRobots.add(url);
- await this.handleSkippedRequest({ url, reason: 'robotsTxt' });
+ for await (const request of requests) {
+ const url = typeof request === 'string' ? request : request.url!;
+
+ if (requestLimit !== undefined && yieldedRequestCount >= requestLimit) {
+ skippedBecauseOfLimit.add(url);
+ continue;
+ }
+
+ if (await isAllowedBasedOnRobotsTxtFile(url)) {
+ yield request;
+ yieldedRequestCount += 1;
+ } else {
+ skippedBecauseOfRobots.add(url);
+ }
}
}
+ const result = await requestQueue.addRequestsBatched(filteredRequests(), options);
+
if (skippedBecauseOfRobots.size > 0) {
this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
skipped: [...skippedBecauseOfRobots],
@@ -1147,11 +1153,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
);
}
- if (!allowedRequests.length) {
- return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
- }
-
- return requestQueue.addRequestsBatched(allowedRequests, options);
+ return result;
}
/**
diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts
index 278c47906..895061a7f 100644
--- a/packages/core/src/storages/request_provider.ts
+++ b/packages/core/src/storages/request_provider.ts
@@ -9,7 +9,14 @@ import type {
RequestQueueInfo,
StorageClient,
} from '@crawlee/types';
-import { chunk, downloadListOfUrls, sleep } from '@crawlee/utils';
+import {
+ chunkedAsyncIterable,
+ downloadListOfUrls,
+ isAsyncIterable,
+ isIterable,
+ peekableAsyncIterable,
+ sleep,
+} from '@crawlee/utils';
import ow from 'ow';
import { ListDictionary, LruCache } from '@apify/datastructures';
@@ -28,6 +35,8 @@ import type { IStorage, StorageManagerOptions } from './storage_manager';
import { StorageManager } from './storage_manager';
import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils';
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+
export abstract class RequestProvider implements IStorage {
id: string;
name?: string;
@@ -202,14 +211,17 @@ export abstract class RequestProvider implements IStorage {
* @param [options] Request queue operation options.
*/
async addRequests(
- requestsLike: Source[],
+ requestsLike: RequestsLike,
options: RequestQueueOperationOptions = {},
): Promise<BatchAddRequestsResult> {
checkStorageAccess();
this.lastActivity = new Date();
- ow(requestsLike, ow.array);
+ ow(
+ requestsLike,
+ ow.object.is((value: unknown) => isIterable(value) || isAsyncIterable(value)),
+ );
ow(
options,
ow.object.exactShape({
@@ -237,19 +249,21 @@ export abstract class RequestProvider implements IStorage {
unprocessedRequests: [],
};
- for (const requestLike of requestsLike) {
- if ('requestsFromUrl' in requestLike) {
- const requests = await this._fetchRequestsFromUrl(requestLike as InternalSource);
- await this._addFetchedRequests(requestLike as InternalSource, requests, options);
+ const requests: Request<Dictionary>[] = [];
+
+ for await (const requestLike of requestsLike) {
+ if (typeof requestLike === 'string') {
+ requests.push(new Request({ url: requestLike }));
+ } else if ('requestsFromUrl' in requestLike) {
+ const fetchedRequests = await this._fetchRequestsFromUrl(requestLike as InternalSource);
+ await this._addFetchedRequests(requestLike as InternalSource, fetchedRequests, options);
+ } else {
+ requests.push(
+ requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions),
+ );
}
}
- const requests = requestsLike
- .filter((requestLike) => !('requestsFromUrl' in requestLike))
- .map((requestLike) => {
- return requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions);
- });
-
const requestsToAdd = new Map<string, Request>();
for (const request of requests) {
@@ -315,7 +329,7 @@ export abstract class RequestProvider implements IStorage {
* @param options Options for the request queue
*/
async addRequestsBatched(
- requests: (string | Source)[],
+ requests: RequestsLike,
options: AddRequestsBatchedOptions = {},
): Promise<AddRequestsBatchedResult> {
checkStorageAccess();
@@ -332,47 +346,48 @@ export abstract class RequestProvider implements IStorage {
}),
);
- // The `requests` array can be huge, and `ow` is very slow for anything more complex.
- // This explicit iteration takes a few milliseconds, while the ow check can take tens of seconds.
-
- // ow(requests, ow.array.ofType(ow.any(
- // ow.string,
- // ow.object.partialShape({ url: ow.string, id: ow.undefined }),
- // ow.object.partialShape({ requestsFromUrl: ow.string, regex: ow.optional.regExp }),
- // )));
-
- for (const request of requests) {
- if (typeof request === 'string') {
- continue;
- }
-
- if (typeof request === 'object' && request !== null) {
- if (typeof request.url === 'string' && typeof request.id === 'undefined') {
- continue;
+ const addRequest = this.addRequest.bind(this);
+
+ async function* generateRequests() {
+ for await (const opts of requests) {
+ // Validate the input
+ if (typeof opts === 'object' && opts !== null) {
+ if (opts.url !== undefined && typeof opts.url !== 'string') {
+ throw new Error(
+ `Request options are not valid, the 'url' property is not a string. Input: ${inspect(opts)}`,
+ );
+ }
+
+ if (opts.id !== undefined) {
+ throw new Error(
+ `Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`,
+ );
+ }
+
+ if (
+ (opts as any).requestsFromUrl !== undefined &&
+ typeof (opts as any).requestsFromUrl !== 'string'
+ ) {
+ throw new Error(
+ `Request options are not valid, the 'requestsFromUrl' property is not a string. Input: ${inspect(opts)}`,
+ );
+ }
}
- if (typeof (request as any).requestsFromUrl === 'string') {
- continue;
+ if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
+ // Handle URL lists right away
+ await addRequest(opts, { forefront: options.forefront });
+ } else {
+ // Yield valid requests
+ yield typeof opts === 'string' ? { url: opts } : (opts as RequestOptions);
}
}
-
- throw new Error(
- `Request options are not valid, provide either a URL or an object with 'url' property (but without 'id' property), or an object with 'requestsFromUrl' property. Input: ${inspect(
- request,
- )}`,
- );
}
const { batchSize = 1000, waitBetweenBatchesMillis = 1000 } = options;
- const sources: Source[] = [];
- for (const opts of requests) {
- if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
- await this.addRequest(opts, { forefront: options.forefront });
- } else {
- sources.push(typeof opts === 'string' ? { url: opts } : (opts as RequestOptions));
- }
- }
+ const chunks = peekableAsyncIterable(chunkedAsyncIterable(generateRequests(), batchSize));
+ const chunksIterator = chunks[Symbol.asyncIterator]();
const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests: Source[], cache = true) => {
const resultsToReturn: ProcessedRequest[] = [];
@@ -395,13 +410,17 @@ export abstract class RequestProvider implements IStorage {
return resultsToReturn;
};
- const initialChunk = sources.splice(0, batchSize);
-
// Add initial batch of `batchSize` to process them right away
+ const initialChunk = await chunksIterator.peek();
+ if (initialChunk === undefined) {
+ return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
+ }
+
const addedRequests = await attemptToAddToQueueAndAddAnyUnprocessed(initialChunk);
+ await chunksIterator.next();
- // If we have no more requests to add, return early
- if (!sources.length) {
+ // If we have no more requests to add, return immediately
+ if ((await chunksIterator.peek()) === undefined) {
return {
addedRequests,
waitForAllRequestsToBeAdded: Promise.resolve([]),
@@ -410,10 +429,9 @@ export abstract class RequestProvider implements IStorage {
// eslint-disable-next-line no-async-promise-executor
const promise = new Promise<ProcessedRequest[]>(async (resolve) => {
- const chunks = chunk(sources, batchSize);
const finalAddedRequests: ProcessedRequest[] = [];
- for (const requestChunk of chunks) {
+ for await (const requestChunk of chunks) {
finalAddedRequests.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(requestChunk, false)));
await sleep(waitBetweenBatchesMillis);
diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.ts
index f66f42bc1..7dd8157d7 100644
--- a/packages/core/src/storages/request_queue_v2.ts
+++ b/packages/core/src/storages/request_queue_v2.ts
@@ -8,6 +8,7 @@ import type {
RequestProviderOptions,
RequestQueueOperationInfo,
RequestQueueOperationOptions,
+ RequestsLike,
} from './request_provider';
import { RequestProvider } from './request_provider';
import { getRequestId } from './utils';
@@ -124,7 +125,7 @@ export class RequestQueue extends RequestProvider {
* @inheritDoc
*/
override async addRequests(
- requestsLike: Source[],
+ requestsLike: RequestsLike,
options: RequestQueueOperationOptions = {},
): Promise<BatchAddRequestsResult> {
const result = await super.addRequests(requestsLike, options);
diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts
index 29d9c91bc..77ff08d88 100644
--- a/packages/utils/src/index.ts
+++ b/packages/utils/src/index.ts
@@ -9,6 +9,7 @@ export * as social from './internals/social';
export * from './internals/typedefs';
export * from './internals/open_graph_parser';
export * from './internals/gotScraping';
+export * from './internals/iterables';
export * from './internals/robots';
export * from './internals/sitemap';
export * from './internals/url';
diff --git a/packages/utils/src/internals/iterables.ts b/packages/utils/src/internals/iterables.ts
new file mode 100644
index 000000000..c81d6e99d
--- /dev/null
+++ b/packages/utils/src/internals/iterables.ts
@@ -0,0 +1,218 @@
+import { inspect } from 'node:util';
+
+/**
+ * Type guard that checks if a value is iterable (has Symbol.iterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isIterable(someValue)) {
+ * for (const item of someValue) {
+ * console.log(item);
+ * }
+ * }
+ * ```
+ */
+export function isIterable<T>(value: unknown): value is Iterable<T> {
+ if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+ return false;
+ }
+
+ if (Array.isArray(value)) {
+ return true;
+ }
+
+ return typeof Object(value)[Symbol.iterator] === 'function';
+}
+
+/**
+ * Type guard that checks if a value is async iterable (has Symbol.asyncIterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isAsyncIterable(someValue)) {
+ * for await (const item of someValue) {
+ * console.log(item);
+ * }
+ * }
+ * ```
+ */
+export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T> {
+ if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+ return false;
+ }
+
+ return typeof Object(value)[Symbol.asyncIterator] === 'function';
+}
+
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ * console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export async function* asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
+ yield* iterable;
+}
+
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ * for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ * console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export async function* chunkedAsyncIterable<T>(
+ iterable: AsyncIterable<T> | Iterable<T>,
+ chunkSize: number,
+): AsyncIterable<T[]> {
+ if (typeof chunkSize !== 'number' || chunkSize < 1) {
+ throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
+ }
+
+ let chunk: T[] = [];
+
+ for await (const item of iterable) {
+ chunk.push(item);
+
+ if (chunk.length >= chunkSize) {
+ yield chunk;
+ chunk = [];
+ }
+ }
+
+ if (chunk.length) {
+ yield chunk;
+ }
+}
+
+/**
+ * An async iterator that also supports peeking at the next value without consuming it.
+ * Extends both AsyncIterator and AsyncIterable interfaces.
+ * @internal
+ */
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+ /**
+ * Peeks at the next value without consuming it from the iterator.
+ * Subsequent calls to peek() will return the same value until next() is called.
+ *
+ * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
+ */
+ peek(): Promise<T | undefined>;
+}
+
+/**
+ * An async iterable that yields peekable async iterators.
+ * @internal
+ */
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+ [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ * yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T> {
+ const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
+ let peekedValue: { done: boolean; value: T } | undefined;
+ let isExhausted = false;
+
+ const peekableIterator: PeekableAsyncIterator<T> = {
+ async next(): Promise<IteratorResult<T>> {
+ // If we have peeked a value, return it and clear the peek
+ if (peekedValue !== undefined) {
+ const result = peekedValue;
+ peekedValue = undefined;
+
+ if (result.done) {
+ isExhausted = true;
+ return { done: true, value: undefined };
+ }
+
+ return { done: false, value: result.value };
+ }
+
+ if (isExhausted) {
+ return { done: true, value: undefined };
+ }
+
+ const result = await iterator.next();
+
+ if (result.done) {
+ isExhausted = true;
+ }
+
+ return result;
+ },
+
+ async peek(): Promise<T | undefined> {
+ if (peekedValue !== undefined) {
+ return peekedValue.done ? undefined : peekedValue.value;
+ }
+
+ if (isExhausted) {
+ return undefined;
+ }
+
+ const result = await iterator.next();
+ peekedValue = { done: result.done ?? false, value: result.value };
+
+ if (result.done) {
+ isExhausted = true;
+ return undefined;
+ }
+
+ return result.value;
+ },
+
+ [Symbol.asyncIterator]() {
+ return this;
+ },
+ };
+
+ return {
+ [Symbol.asyncIterator]() {
+ return peekableIterator;
+ },
+ };
+}
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 87d8de272..02c6df89a 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1463,10 +1463,12 @@ describe('BasicCrawler', () => {
await crawler.addRequests(requestsToAdd);
// Should only have added the first 3 requests (since 2 were already processed, limit allows 3 more)
- expect(addRequestsBatchedSpy).toHaveBeenCalledWith(
- ['http://example.com/1', 'http://example.com/2', 'http://example.com/3'],
- {},
- );
+ expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
+ await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
+ { url: 'http://example.com/1' },
+ { url: 'http://example.com/2' },
+ { url: 'http://example.com/3' },
+ ]);
});
test('should not enqueue more requests than maxRequestsPerCrawl allows across multiple addRequests calls', async () => {
@@ -1484,6 +1486,11 @@ describe('BasicCrawler', () => {
// First call - should add 2 requests (2 more slots to go)
await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);
+ await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
+ { url: 'http://example.com/1' },
+ { url: 'http://example.com/2' },
+ ]);
+
// Second call - should add only 2 more requests
await crawler.addRequests([
'http://example.com/3',
@@ -1492,14 +1499,24 @@ describe('BasicCrawler', () => {
'http://example.com/6', // This should be ignored
]);
+ await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
+ { url: 'http://example.com/1' },
+ { url: 'http://example.com/2' },
+ { url: 'http://example.com/3' },
+ { url: 'http://example.com/4' },
+ ]);
+
// Third call - should add no requests (limit already reached)
await crawler.addRequests(['http://example.com/7', 'http://example.com/8']);
- expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2); // Third call shouldn't trigger addRequestsBatched
- expect(addRequestsBatchedSpy.mock.calls).toEqual([
- [['http://example.com/1', 'http://example.com/2'], {}],
- [['http://example.com/3', 'http://example.com/4'], {}],
+ await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
+ { url: 'http://example.com/1' },
+ { url: 'http://example.com/2' },
+ { url: 'http://example.com/3' },
+ { url: 'http://example.com/4' },
]);
+
+ expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(3);
});
test('should respect robots.txt when limiting requests', async () => {
@@ -1527,8 +1544,13 @@ describe('BasicCrawler', () => {
'http://example.com/4', // Would exceed limit
]);
+ await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
+ { url: 'http://example.com/1' },
+ { url: 'http://example.com/3' },
+ ]);
+
// Should only have added the first request (allowed by robots.txt and within limit)
- expect(addRequestsBatchedSpy).toHaveBeenCalledWith(['http://example.com/1', 'http://example.com/3'], {});
+ expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
});
test('enqueueLinks should respect maxRequestsPerCrawl', async () => {
@@ -1544,11 +1566,14 @@ describe('BasicCrawler', () => {
'http://example.com/5',
'http://example.com/6',
];
+ const visitedUrls: string[] = [];
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 5,
requestHandler: async (context) => {
+ visitedUrls.push(context.request.url);
+
if (context.request.label) {
return;
}
@@ -1561,12 +1586,14 @@ describe('BasicCrawler', () => {
await crawler.run(['http://example.com']);
+ expect(visitedUrls).toEqual([
+ 'http://example.com', // added by crawler.run()
+ 'http://example.com/1',
+ 'http://example.com/2',
+ ]);
+
// Should only have added the first 2 requests (since 2 were already processed and 1 is in progress, limit allows 2 more)
expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2);
- expect(addRequestsBatchedSpy.mock.calls).toMatchObject([
- [['http://example.com'], {}], // added by crawler.run()
- [[{ url: 'http://example.com/1' }, { url: 'http://example.com/2' }], {}],
- ]);
});
});
diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.ts
index cdbe340f8..05321bf78 100644
--- a/test/core/enqueue_links/click_elements.test.ts
+++ b/test/core/enqueue_links/click_elements.test.ts
@@ -31,8 +31,12 @@ function createRequestQueueMock() {
// @ts-expect-error Override method for testing
requestQueue.addRequests = async function (requests) {
- enqueued.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
+ const processedRequests: Source[] = [];
+ for await (const request of requests) {
+ processedRequests.push(typeof request === 'string' ? { url: request } : request);
+ }
+ enqueued.push(...processedRequests);
+ return { processedRequests, unprocessedRequests: [] as never[] };
};
return { enqueued, requestQueue };
@@ -113,7 +117,9 @@ testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => {
const addedRequests: { request: Source; options?: RequestQueueOperationOptions }[] = [];
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
requestQueue.addRequests = async (requests, options) => {
- addedRequests.push(...requests.map((request) => ({ request, options })));
+ for await (const request of requests) {
+ addedRequests.push({ request: typeof request === 'string' ? { url: request } : request, options });
+ }
return { processedRequests: [], unprocessedRequests: [] };
};
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 2b33d8d1d..fbf34c604 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -8,7 +8,7 @@ import {
launchPuppeteer,
RequestQueue,
} from '@crawlee/puppeteer';
-import type { CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils';
import { load } from 'cheerio';
import type { Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright';
import type { Browser as PuppeteerBrowser, Page as PuppeteerPage } from 'puppeteer';
@@ -50,8 +50,12 @@ function createRequestQueueMock() {
// @ts-expect-error Override method for testing
requestQueue.addRequests = async function (requests) {
- enqueued.push(...requests);
- return { processedRequests: requests, unprocessedRequests: [] as never[] };
+ const processedRequests: Source[] = [];
+ for await (const request of requests) {
+ processedRequests.push(typeof request === 'string' ? { url: request } : request);
+ }
+ enqueued.push(...processedRequests);
+ return { processedRequests, unprocessedRequests: [] as never[] };
};
return { enqueued, requestQueue };
@@ -972,7 +976,9 @@ describe('enqueueLinks()', () => {
requestQueue.addRequests = async (requests, options) => {
// copy the requests to the enqueued list, along with options that were passed to addRequests,
// so that it doesn't matter how many calls were made
- enqueued.push(...requests.map((request) => ({ request, options })));
+ for await (const request of requests) {
+ enqueued.push({ request: typeof request === 'string' ? { url: request } : request, options });
+ }
return { processedRequests: [], unprocessedRequests: [] };
};
@@ -999,7 +1005,9 @@ describe('enqueueLinks()', () => {
requestQueue.addRequestsBatched = async (requests, options) => {
// copy the requests to the enqueued list, along with options that were passed to addRequests,
// so that it doesn't matter how many calls were made
- enqueued.push(...requests.map((request) => ({ request, options })));
+ for await (const request of requests) {
+ enqueued.push({ request: typeof request === 'string' ? { url: request } : request, options });
+ }
return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
};
diff --git a/test/utils/iterables.test.ts b/test/utils/iterables.test.ts
new file mode 100644
index 000000000..f571277c2
--- /dev/null
+++ b/test/utils/iterables.test.ts
@@ -0,0 +1,202 @@
+import { asyncifyIterable, chunkedAsyncIterable, peekableAsyncIterable } from '@crawlee/utils';
+import { describe, expect, it } from 'vitest';
+
+describe('asyncifyIterable', () => {
+ it('should convert a regular array to async iterable', async () => {
+ const asyncIterable = asyncifyIterable([1, 2, 3, 4, 5]);
+
+ const result = [];
+ for await (const item of asyncIterable) {
+ result.push(item);
+ }
+
+ expect(result).toEqual([1, 2, 3, 4, 5]);
+ });
+
+ it('should handle empty arrays', async () => {
+ const asyncIterable = asyncifyIterable([]);
+
+ const result: unknown[] = [];
+ for await (const item of asyncIterable) {
+ result.push(item);
+ }
+
+ expect(result).toEqual([]);
+ });
+
+ it('should work with Set', async () => {
+ const asyncIterable = asyncifyIterable(new Set([1, 2, 3, 2, 1]));
+
+ const result = [];
+ for await (const item of asyncIterable) {
+ result.push(item);
+ }
+
+ expect(result).toEqual([1, 2, 3]);
+ });
+
+ it('should work with generator function', async () => {
+ function* generator() {
+ yield 1;
+ yield 2;
+ yield 3;
+ }
+
+ const asyncIterable = asyncifyIterable(generator());
+
+ const result = [];
+ for await (const item of asyncIterable) {
+ result.push(item);
+ }
+
+ expect(result).toEqual([1, 2, 3]);
+ });
+});
+
+describe('chunkedAsyncIterable', () => {
+ it('should chunk an async iterable into specified sizes', async () => {
+ const result = [];
+ for await (const chunk of chunkedAsyncIterable([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3)) {
+ result.push(chunk);
+ }
+
+ expect(result).toEqual([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]]);
+ });
+
+ it('should handle chunk size of 1', async () => {
+ const result = [];
+ for await (const chunk of chunkedAsyncIterable([1, 2, 3], 1)) {
+ result.push(chunk);
+ }
+
+ expect(result).toEqual([[1], [2], [3]]);
+ });
+
+ it('should handle chunk size larger than input', async () => {
+ const chunked = chunkedAsyncIterable([1, 2, 3], 10);
+
+ const result = [];
+ for await (const chunk of chunked) {
+ result.push(chunk);
+ }
+
+ expect(result).toEqual([[1, 2, 3]]);
+ });
+
+ it('should handle empty async iterable', async () => {
+ const result = [];
+ for await (const chunk of chunkedAsyncIterable([], 3)) {
+ result.push(chunk);
+ }
+
+ expect(result).toEqual([]);
+ });
+
+ it('should throw error for invalid chunk size', async () => {
+ await expect(
+ (async () => {
+ for await (const _ of chunkedAsyncIterable([1, 2, 3], 0)) {
+ // Empty block
+ }
+ })(),
+ ).rejects.toThrow();
+ await expect(
+ (async () => {
+ for await (const _ of chunkedAsyncIterable([1, 2, 3], -1)) {
+ // Empty block
+ }
+ })(),
+ ).rejects.toThrow();
+ });
+});
+
+describe('peekableAsyncIterable', () => {
+ it('should allow peeking at the next value without advancing', async () => {
+ const iterable = peekableAsyncIterable([1, 2, 3]);
+ const iterator = iterable[Symbol.asyncIterator]();
+
+ const peeked = await iterator.peek();
+ expect(peeked).toBe(1);
+
+ // Peeking again should return the same value
+ const peekedAgain = await iterator.peek();
+ expect(peekedAgain).toBe(1);
+
+ // Now iterate and verify we get the peeked value first
+ const results = [];
+ for await (const value of iterable) {
+ results.push(value);
+ }
+
+ expect(results).toEqual([1, 2, 3]);
+ });
+
+ it('should return undefined when peeking at empty iterable', async () => {
+ const iterable = peekableAsyncIterable([]);
+ const iterator = iterable[Symbol.asyncIterator]();
+
+ const peeked = await iterator.peek();
+ expect(peeked).toBeUndefined();
+ });
+
+ it('should handle peek after exhausting the iterable', async () => {
+ const iterable = peekableAsyncIterable([1]);
+
+ // Consume the iterable
+ const results = [];
+ for await (const value of iterable) {
+ results.push(value);
+ }
+ expect(results).toEqual([1]);
+
+ // Get a fresh iterator and peek should return undefined
+ const iterator = iterable[Symbol.asyncIterator]();
+ const peeked = await iterator.peek();
+ expect(peeked).toBeUndefined();
+ });
+
+ it('should work with manual iteration', async () => {
+ const iterable = peekableAsyncIterable([10, 20, 30]);
+ const iterator = iterable[Symbol.asyncIterator]();
+
+ // Peek first
+ expect(await iterator.peek()).toBe(10);
+
+ const first = await iterator.next();
+ expect(first.value).toBe(10);
+ expect(first.done).toBe(false);
+ expect(await iterator.peek()).toBe(20);
+
+ const second = await iterator.next();
+ expect(second.value).toBe(20);
+ expect(second.done).toBe(false);
+ expect(await iterator.peek()).toBe(30);
+
+ const third = await iterator.next();
+ expect(third.value).toBe(30);
+ expect(third.done).toBe(false);
+ expect(await iterator.peek()).toBe(undefined);
+
+ const done = await iterator.next();
+ expect(done.done).toBe(true);
+ expect(await iterator.peek()).toBe(undefined);
+ });
+
+ it('should handle peek on single element iterable', async () => {
+ const iterable = peekableAsyncIterable([42]);
+ const iterator = iterable[Symbol.asyncIterator]();
+
+ expect(await iterator.peek()).toBe(42);
+
+ const results = [];
+ for await (const value of iterable) {
+ results.push(value);
+ }
+
+ expect(results).toEqual([42]);
+
+ // Get a fresh iterator and peek after exhaustion
+ const newIterator = iterable[Symbol.asyncIterator]();
+ expect(await newIterator.peek()).toBeUndefined();
+ });
+});