Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `TandemRequestProvider` for combined `RequestList` and `RequestQueue` usage (#2914)

हिमांशु4ca450f08b9f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index cd0a22bf8..bf1b75c5b 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -12,6 +12,7 @@ import type {
FinalStatistics,
GetUserDataFromRequest,
IRequestList,
+ IRequestManager,
LoadedContext,
ProxyInfo,
Request,
@@ -38,6 +39,8 @@ import {
mergeCookies,
NonRetryableError,
purgeDefaultStorages,
+ RequestListAdapter,
+ RequestManagerTandem,
RequestProvider,
RequestQueue,
RequestQueueV1,
@@ -185,6 +188,14 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
*/
requestQueue?: RequestProvider;
+ /**
+ * Allows explicitly configuring a request manager. Mutually exclusive with the `requestQueue` and `requestList` options.
+ *
+ * This enables explicitly configuring the crawler to use `RequestManagerTandem`, for instance.
+ * If using this, the type of `BasicCrawler.requestQueue` may not be fully compatible with the `RequestProvider` class.
+ */
+ requestManager?: IRequestManager;
+
/**
* Timeout in which the function passed as {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
* @default 60
@@ -492,6 +503,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
*/
requestQueue?: RequestProvider;
+ /**
+ * The main request-handling component of the crawler. It's initialized during the crawler startup.
+ */
+ protected requestManager?: IRequestManager;
+
/**
* A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.
* Only available if used by the crawler.
@@ -605,6 +621,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const {
requestList,
requestQueue,
+ requestManager,
maxRequestRetries = 3,
sameDomainDelaySecs = 0,
maxSessionRotations = 10,
@@ -647,8 +664,19 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
httpClient,
} = options;
- this.requestList = requestList;
- this.requestQueue = requestQueue;
+ if (requestManager !== undefined) {
+ if (requestList !== undefined || requestQueue !== undefined) {
+ throw new Error(
+ 'The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`',
+ );
+ }
+ this.requestManager = requestManager;
+ this.requestQueue = requestManager as RequestProvider; // TODO(v4) - the cast is not fully legitimate here, but it's fine for internal usage by the BasicCrawler
+ } else {
+ this.requestList = requestList;
+ this.requestQueue = requestQueue;
+ }
+
this.httpClient = httpClient ?? new GotScrapingHttpClient();
this.log = log;
this.statusMessageLoggingInterval = statusMessageLoggingInterval;
@@ -884,7 +912,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed
} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
} else {
- const total = this.requestQueue?.getTotalCount() || this.requestList?.length();
+ const total = this.requestManager?.getTotalCount();
message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${
this.stats.state.requestsFailed
} failed requests, desired concurrency ${this.autoscaledPool?.desiredConcurrency ?? 0}.`;
@@ -934,6 +962,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
await this.requestQueue.drop();
this.requestQueue = await this._getRequestQueue();
+ this.requestManager = undefined;
+ await this.initializeRequestManager();
this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests
}
@@ -1047,16 +1077,26 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
});
}
- async getRequestQueue() {
+ async getRequestQueue(): Promise<RequestProvider> {
if (!this.requestQueue && this.requestList) {
this.log.warningOnce(
'When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.',
);
}
- this.requestQueue ??= await this._getRequestQueue();
+ if (!this.requestQueue) {
+ this.requestQueue = await this._getRequestQueue();
+ this.requestManager = undefined;
+ }
+
+ if (!this.requestManager) {
+ this.requestManager =
+ this.requestList === undefined
+ ? this.requestQueue
+ : new RequestManagerTandem(this.requestList, this.requestQueue);
+ }
- return this.requestQueue!;
+ return this.requestQueue;
}
async useState<State extends Dictionary = Dictionary>(defaultValue = {} as State): Promise<State> {
@@ -1065,16 +1105,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
protected get pendingRequestCountApproximation(): number {
- let result = 0;
-
- if (this.requestQueue) {
- result += this.requestQueue.getPendingCount();
- }
- if (this.requestList) {
- result += this.requestList.length() - this.requestList.handledCount();
- }
-
- return result;
+ return this.requestManager?.getPendingCount() ?? 0;
}
protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined {
@@ -1117,7 +1148,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
requests: RequestsLike,
options: CrawlerAddRequestsOptions = {},
): Promise<CrawlerAddRequestsResult> {
- const requestQueue = await this.getRequestQueue();
+ await this.getRequestQueue();
const requestLimit = this.calculateEnqueuedRequestLimit();
@@ -1160,7 +1191,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
}
- const result = await requestQueue.addRequestsBatched(filteredRequests(), options);
+ const result = await this.requestManager!.addRequestsBatched(filteredRequests(), options);
if (skippedBecauseOfRobots.size > 0) {
this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
@@ -1270,6 +1301,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
return items;
}
+ /**
+ * Initializes the crawler.
+ */
protected async _init(): Promise<void> {
if (!this.events.isInitialized()) {
await this.events.init();
@@ -1287,6 +1321,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.sessionPool.setMaxListeners(20);
}
+ await this.initializeRequestManager();
await this._loadHandledRequestCount();
}
@@ -1377,32 +1412,37 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
/**
- * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
- * and RequestQueue is present then enqueues it to the queue first.
+ * Initializes the RequestManager based on the configured requestList and requestQueue.
*/
- protected async _fetchNextRequest() {
- if (!this.requestList || (await this.requestList.isFinished())) {
- return this.requestQueue?.fetchNextRequest();
+ private async initializeRequestManager() {
+ if (this.requestManager !== undefined) {
+ return;
}
- const request = await this.requestList.fetchNextRequest();
- if (!this.requestQueue) return request;
- if (!request) return this.requestQueue.fetchNextRequest();
+ if (this.requestList && this.requestQueue) {
+ // Create a RequestManagerTandem if both RequestList and RequestQueue are provided
+ this.requestManager = new RequestManagerTandem(this.requestList, this.requestQueue);
+ } else if (this.requestQueue) {
+ // Use RequestQueue directly if only it is provided
+ this.requestManager = this.requestQueue;
+ } else if (this.requestList) {
+ // Use RequestList directly if only it is provided
+ // Make it compatible with the IRequestManager interface
+ this.requestManager = new RequestListAdapter(this.requestList);
+ }
- try {
- await this.requestQueue.addRequest(request, { forefront: true });
- } catch (err) {
- // If requestQueue.addRequest() fails here then we must reclaim it back to
- // the RequestList because probably it's not yet in the queue!
- this.log.error(
- 'Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.',
- { request },
- );
- await this.requestList.reclaimRequest(request);
- return null;
+ // If neither RequestList nor RequestQueue is provided, leave the requestManager uninitialized until `getRequestQueue` is called
+ }
+
+ /**
+ * Fetches the next request to process from the underlying request provider.
+ */
+ protected async _fetchNextRequest() {
+ if (this.requestManager === undefined) {
+ throw new Error(`_fetchNextRequest called on an uninitialized crawler`);
}
- await this.requestList.markRequestHandled(request);
- return this.requestQueue.fetchNextRequest();
+
+ return this.requestManager.fetchNextRequest();
}
/**
@@ -1416,7 +1456,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* adding it back to the queue after the timeout passes. Returns `true` if the request
* should be ignored and will be reclaimed to the queue once ready.
*/
- protected delayRequest(request: Request, source: IRequestList | RequestProvider) {
+ protected delayRequest(request: Request, source: IRequestList | RequestProvider | IRequestManager) {
const domain = getDomain(request.url);
if (!domain || !request) {
@@ -1459,7 +1499,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* then retries them in a case of an error, etc.
*/
protected async _runTaskFunction() {
- const source = this.requestQueue || this.requestList || (await this.getRequestQueue());
+ const source = this.requestManager;
+ if (!source) throw new Error('Request provider is not initialized!');
let request: Request | null | undefined;
let session: Session | undefined;
@@ -1521,9 +1562,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
request,
session,
enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {
+ await this.getRequestQueue();
+
return enqueueLinks({
// specify the RQ first to allow overriding it
- requestQueue: await this.getRequestQueue(),
+ requestQueue: this.requestManager!,
robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),
onSkippedRequest: this.handleSkippedRequest,
limit: this.calculateEnqueuedRequestLimit(options.limit),
@@ -1667,24 +1710,14 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* Returns true if either RequestList or RequestQueue have a request ready for processing.
*/
protected async _isTaskReadyFunction() {
- // First check RequestList, since it's only in memory.
- const isRequestListEmpty = this.requestList ? await this.requestList.isEmpty() : true;
- // If RequestList is not empty, task is ready, no reason to check RequestQueue.
- if (!isRequestListEmpty) return true;
- // If RequestQueue is not empty, task is ready, return true, otherwise false.
- return this.requestQueue ? !(await this.requestQueue.isEmpty()) : false;
+ return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
}
/**
* Returns true if both RequestList and RequestQueue have all requests finished.
*/
protected async _defaultIsFinishedFunction() {
- const [isRequestListFinished, isRequestQueueFinished] = await Promise.all([
- this.requestList ? this.requestList.isFinished() : true,
- this.requestQueue ? this.requestQueue.isFinished() : true,
- ]);
- // If both are finished, return true, otherwise return false.
- return isRequestListFinished && isRequestQueueFinished;
+ return !this.requestManager || (await this.requestManager.isFinished());
}
private async _rotateSession(crawlingContext: Context) {
@@ -1701,7 +1734,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected async _requestFunctionErrorHandler(
error: Error,
crawlingContext: Context,
- source: IRequestList | RequestProvider,
+ source: IRequestList | IRequestManager,
): Promise<void> {
const { request } = crawlingContext;
request.pushErrorMessage(error);
@@ -1844,18 +1877,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
/**
- * Updates handledRequestsCount from possibly stored counts,
- * usually after worker migration. Since one of the stores
- * needs to have priority when both are present,
- * it is the request queue, because generally, the request
- * list will first be dumped into the queue and then left
- * empty.
+ * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
*/
protected async _loadHandledRequestCount(): Promise<void> {
- if (this.requestQueue) {
- this.handledRequestsCount = await this.requestQueue.handledCount();
- } else if (this.requestList) {
- this.handledRequestsCount = this.requestList.handledCount();
+ if (this.requestManager) {
+ this.handledRequestsCount = await this.requestManager.handledCount();
}
}
diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts
index 60ef3bc48..ebe9eb2ea 100644
--- a/packages/core/src/storages/index.ts
+++ b/packages/core/src/storages/index.ts
@@ -1,6 +1,7 @@
export * from './dataset';
export * from './key_value_store';
export * from './request_list';
+export * from './request_list_adapter';
export * from './request_provider';
export { RequestQueueV1 } from './request_queue';
export { RequestQueue } from './request_queue_v2';
@@ -9,3 +10,4 @@ export * from './storage_manager';
export * from './utils';
export * from './access_checking';
export * from './sitemap_request_list';
+export * from './request_manager_tandem';
diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts
index eba5b54bf..17abbcab7 100644
--- a/packages/core/src/storages/request_list.ts
+++ b/packages/core/src/storages/request_list.ts
@@ -66,12 +66,6 @@ export interface IRequestList {
fetchNextRequest(): Promise<Request | null>;
/**
- * Gets the next {@apilink Request} to process. First, the function gets a request previously reclaimed
- * using the {@apilink RequestList.reclaimRequest} function, if there is any.
- * Otherwise it gets the next request from sources.
- *
- * The function resolves to `null` if there are no more requests to process.
- *
* Can be used to iterate over the `RequestList` instance in a `for await .. of` loop.
* Provides an alternative for the repeated use of `fetchNextRequest`.
*/
diff --git a/packages/core/src/storages/request_list_adapter.ts b/packages/core/src/storages/request_list_adapter.ts
new file mode 100644
index 000000000..0e39dea17
--- /dev/null
+++ b/packages/core/src/storages/request_list_adapter.ts
@@ -0,0 +1,102 @@
+import type { Dictionary } from '@crawlee/types';
+
+import type { Request } from '../request';
+import type { IRequestList } from './request_list';
+import type {
+ AddRequestsBatchedResult,
+ IRequestManager,
+ RequestQueueOperationInfo,
+ RequestQueueOperationOptions,
+} from './request_provider';
+
+/**
+ * Adapts the IRequestList interface to the IRequestManager interface.
+ * It simply throws an exception when inserting requests is attempted.
+ * @internal
+ */
+export class RequestListAdapter implements IRequestManager {
+ constructor(private requestList: IRequestList) {}
+
+ /**
+ * @inheritdoc
+ */
+ async isFinished(): Promise<boolean> {
+ return this.requestList.isFinished();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async isEmpty(): Promise<boolean> {
+ return this.requestList.isEmpty();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async handledCount(): Promise<number> {
+ return Promise.resolve(this.requestList.handledCount());
+ }
+
+ /**
+ * @inheritdoc
+ */
+ getTotalCount(): number {
+ return this.requestList.length();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ getPendingCount(): number {
+ return this.requestList.length() - this.requestList.handledCount();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null> {
+ return await this.requestList.fetchNextRequest();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async markRequestHandled(request: Request): Promise<void> {
+ return this.requestList.markRequestHandled(request);
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async reclaimRequest(
+ request: Request,
+ _options?: RequestQueueOperationOptions,
+ ): Promise<RequestQueueOperationInfo | null> {
+ await this.requestList.reclaimRequest(request);
+ return null;
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async *[Symbol.asyncIterator]() {
+ for await (const request of this.requestList) {
+ yield request;
+ }
+ }
+
+ /**
+ * @inheritdoc
+ */
+ addRequestsBatched(): Promise<AddRequestsBatchedResult> {
+ throw new Error('Cannot add requests to a read-only request list');
+ }
+
+ /**
+ * @inheritdoc
+ */
+ addRequest(): Promise<RequestQueueOperationInfo> {
+ throw new Error('Cannot add requests to a read-only request list');
+ }
+}
diff --git a/packages/core/src/storages/request_manager_tandem.ts b/packages/core/src/storages/request_manager_tandem.ts
new file mode 100644
index 000000000..cc79cf45c
--- /dev/null
+++ b/packages/core/src/storages/request_manager_tandem.ts
@@ -0,0 +1,163 @@
+import type { Dictionary } from '@crawlee/types';
+
+import type { Log } from '@apify/log';
+
+import { log } from '../log';
+import type { Request, Source } from '../request';
+import type { IRequestList } from './request_list';
+import type {
+ AddRequestsBatchedOptions,
+ AddRequestsBatchedResult,
+ IRequestManager,
+ RequestQueueOperationInfo,
+ RequestQueueOperationOptions,
+ RequestsLike,
+} from './request_provider';
+
+/**
+ * A request manager that combines a RequestList and a RequestQueue.
+ * It first reads requests from the RequestList and then, when needed,
+ * transfers them in batches to the RequestQueue.
+ */
+export class RequestManagerTandem implements IRequestManager {
+ private log: Log;
+ private requestList: IRequestList;
+ private requestQueue: IRequestManager;
+
+ constructor(requestList: IRequestList, requestQueue: IRequestManager) {
+ this.log = log.child({ prefix: 'RequestManagerTandem' });
+ this.requestList = requestList;
+ this.requestQueue = requestQueue;
+ }
+
+ /**
+ * Transfers a batch of requests from the RequestList to the RequestQueue.
+ * Handles both successful transfers and failures appropriately.
+ * @private
+ */
+ private async transferNextBatchToQueue(): Promise<void> {
+ const request = await this.requestList.fetchNextRequest();
+
+ if (request === null) {
+ return;
+ }
+
+ try {
+ await this.requestQueue.addRequest(request, { forefront: true });
+ } catch (error) {
+ // If requestQueue.addRequest() fails here then we must reclaim it back to
+ // the RequestList because probably it's not yet in the queue!
+ this.log.error(
+ 'Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.',
+ { request },
+ );
+ await this.requestList.reclaimRequest(request);
+ return;
+ }
+
+ await this.requestList.markRequestHandled(request);
+ }
+
+ /**
+ * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
+ * is not finished, it will transfer a batch of requests from the list to the queue first.
+ * @inheritdoc
+ */
+ async fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null> {
+ // First, try to transfer a request from the requestList
+ const [listEmpty, listFinished] = await Promise.all([
+ this.requestList.isEmpty(),
+ this.requestList.isFinished(),
+ ]);
+
+ if (!listEmpty && !listFinished) {
+ await this.transferNextBatchToQueue();
+ }
+
+ // Try to fetch from queue after potential transfer
+ return this.requestQueue.fetchNextRequest<T>();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async isFinished(): Promise<boolean> {
+ const storagesFinished = await Promise.all([this.requestList.isFinished(), this.requestQueue.isFinished()]);
+ return storagesFinished.every(Boolean);
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async isEmpty(): Promise<boolean> {
+ const storagesEmpty = await Promise.all([this.requestList.isEmpty(), this.requestQueue.isEmpty()]);
+ return storagesEmpty.every(Boolean);
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async handledCount(): Promise<number> {
+ // Since one of the stores needs to have priority when both are present, we query the request queue - the request list will first be dumped into the queue and then left empty.
+ return await this.requestQueue.handledCount();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ getTotalCount(): number {
+ return this.requestQueue.getTotalCount();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ getPendingCount(): number {
+ return this.requestQueue.getPendingCount() + this.requestList.length() - this.requestList.handledCount();
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async *[Symbol.asyncIterator]() {
+ while (true) {
+ const req = await this.fetchNextRequest();
+ if (!req) break;
+ yield req;
+ }
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null> {
+ return this.requestQueue.markRequestHandled(request);
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async reclaimRequest(
+ request: Request,
+ options?: RequestQueueOperationOptions,
+ ): Promise<RequestQueueOperationInfo | null> {
+ return await this.requestQueue.reclaimRequest(request, options);
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo> {
+ return await this.requestQueue.addRequest(requestLike, options);
+ }
+
+ /**
+ * @inheritdoc
+ */
+ async addRequestsBatched(
+ requests: RequestsLike,
+ options?: AddRequestsBatchedOptions,
+ ): Promise<AddRequestsBatchedResult> {
+ return await this.requestQueue.addRequestsBatched(requests, options);
+ }
+}
diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts
index 41468fc58..9c4df9712 100644
--- a/packages/core/src/storages/request_provider.ts
+++ b/packages/core/src/storages/request_provider.ts
@@ -38,7 +38,68 @@ import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './uti
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
-export abstract class RequestProvider implements IStorage {
+/**
+ * Represents a provider of requests/URLs to crawl.
+ */
+export interface IRequestManager {
+ /**
+ * Returns `true` if all requests were already handled and there are no more left.
+ */
+ isFinished(): Promise<boolean>;
+
+ /**
+ * Resolves to `true` if the next call to {@apilink IRequestManager.fetchNextRequest} function
+ * would return `null`, otherwise it resolves to `false`.
+ * Note that even if the provider is empty, there might be some pending requests currently being processed.
+ */
+ isEmpty(): Promise<boolean>;
+
+ /**
+ * Returns number of handled requests.
+ */
+ handledCount(): Promise<number>;
+
+ /**
+ * Get the total number of requests known to the request manager.
+ */
+ getTotalCount(): number;
+
+ /**
+ * Get an offline approximation of the number of pending requests.
+ */
+ getPendingCount(): number;
+
+ /**
+ * Gets the next {@apilink Request} to process.
+ *
+ * The function's `Promise` resolves to `null` if there are no more
+ * requests to process.
+ */
+ fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+
+ /**
+ * Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
+ * Provides an alternative for the repeated use of `fetchNextRequest`.
+ */
+ [Symbol.asyncIterator](): AsyncGenerator<Request>;
+
+ /**
+ * Marks request as handled after successful processing.
+ */
+ markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+
+ /**
+ * Reclaims request to the provider if its processing failed.
+ * The request will become available in the next `fetchNextRequest()`.
+ */
+ reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+
+ addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+
+ addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+}
+
+export abstract class RequestProvider implements IStorage, IRequestManager {
id: string;
name?: string;
timeoutSecs = 30;
@@ -499,7 +560,7 @@ export abstract class RequestProvider implements IStorage {
* @returns
* Returns the request object or `null` if there are no more pending requests.
*/
- abstract fetchNextRequest<T extends Dictionary = Dictionary>(options?: RequestOptions): Promise<Request<T> | null>;
+ abstract fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
/**
* Marks a request that was previously returned by the
@@ -661,6 +722,17 @@ export abstract class RequestProvider implements IStorage {
manager.closeStorage(this);
}
+ /**
+ * @inheritdoc
+ */
+ async *[Symbol.asyncIterator]() {
+ while (true) {
+ const req = await this.fetchNextRequest();
+ if (!req) break;
+ yield req;
+ }
+ }
+
/**
* Returns the number of handled requests.
*
@@ -669,6 +741,7 @@ export abstract class RequestProvider implements IStorage {
* ```javascript
* const { handledRequestCount } = await queue.getInfo();
* ```
+ * @inheritdoc
*/
async handledCount(): Promise<number> {
// NOTE: We keep this function for compatibility with RequestList.handledCount()
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index bce9bbd9e..216c57c5d 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -767,27 +767,16 @@ describe('BasicCrawler', () => {
const request1 = new Request({ id: 'id-1', ...sources[1] });
const request2 = new Request({ id: 'id-2', ...sources[2] });
- vitest
- .spyOn(requestQueue, 'fetchNextRequest')
- .mockResolvedValueOnce(request0)
- .mockResolvedValueOnce(request1)
- .mockResolvedValueOnce(request2)
- .mockResolvedValueOnce(request1)
- .mockResolvedValueOnce(request1)
- .mockResolvedValueOnce(request1);
+ const queueContent = [request0, request1, request2, request1, request1, request1];
+
+ vitest.spyOn(requestQueue, 'fetchNextRequest').mockImplementation(async () => queueContent.shift() ?? null);
const markReqHandled = vitest
.spyOn(requestQueue, 'markRequestHandled')
.mockReturnValue(Promise.resolve() as any);
const reclaimReq = vitest.spyOn(requestQueue, 'reclaimRequest').mockReturnValue(Promise.resolve() as any);
- vitest
- .spyOn(requestQueue, 'isEmpty')
- .mockResolvedValueOnce(false)
- .mockResolvedValueOnce(false)
- .mockResolvedValueOnce(false)
- .mockResolvedValueOnce(true);
-
+ vitest.spyOn(requestQueue, 'isEmpty').mockImplementation(async () => queueContent.length <= 0);
vitest.spyOn(requestQueue, 'isFinished').mockResolvedValueOnce(true);
await basicCrawler.run();
diff --git a/test/core/request_manager_tandem.test.ts b/test/core/request_manager_tandem.test.ts
new file mode 100644
index 000000000..1117de5c6
--- /dev/null
+++ b/test/core/request_manager_tandem.test.ts
@@ -0,0 +1,235 @@
+import { log, Request, RequestList, RequestManagerTandem, RequestQueue } from '@crawlee/core';
+import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
+
+import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator';
+
+describe('RequestManagerTandem', () => {
+ let logLevel: number;
+ const emulator = new MemoryStorageEmulator();
+
+ beforeAll(() => {
+ logLevel = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ beforeEach(async () => {
+ await emulator.init();
+ vi.restoreAllMocks();
+ });
+
+ afterAll(async () => {
+ log.setLevel(logLevel);
+ await emulator.destroy();
+ });
+
+ test('fetchNextRequest transfers from list to queue when queue is empty', async () => {
+ // Create sources with 3 URLs
+ const requestList = await RequestList.open(null, [
+ { url: 'https://example.com/1' },
+ { url: 'https://example.com/2' },
+ { url: 'https://example.com/3' },
+ ]);
+ const requestQueue = await RequestQueue.open();
+
+ // Mock the addRequest function of requestQueue to verify it's called
+ const addRequestSpy = vi.spyOn(requestQueue, 'addRequest');
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // First fetch should trigger transfer from list to queue
+ const request1 = await tandem.fetchNextRequest();
+
+ // Verify the request was transferred from list to queue
+ expect(addRequestSpy).toHaveBeenCalled();
+ expect(request1).not.toBeNull();
+ expect(request1?.url).toBe('https://example.com/1');
+
+ // Fetch more requests to ensure they all come from the queue
+ const request2 = await tandem.fetchNextRequest();
+ expect(request2).not.toBeNull();
+ expect(request2?.url).toBe('https://example.com/2');
+
+ const request3 = await tandem.fetchNextRequest();
+ expect(request3).not.toBeNull();
+ expect(request3?.url).toBe('https://example.com/3');
+
+ // No more requests should be available
+ const request4 = await tandem.fetchNextRequest();
+ expect(request4).toBeNull();
+ });
+
+ test('markRequestHandled properly marks request as handled in the queue', async () => {
+ const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);
+ const requestQueue = await RequestQueue.open();
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Mock markRequestHandled in requestQueue
+ const markHandledSpy = vi.spyOn(requestQueue, 'markRequestHandled');
+
+ // First fetch a request
+ const request = await tandem.fetchNextRequest();
+ expect(request).not.toBeNull();
+
+ // Mark it as handled
+ await tandem.markRequestHandled(request!);
+
+ // Verify the queue's markRequestHandled was called
+ expect(markHandledSpy).toHaveBeenCalledWith(request);
+ });
+
+ test('reclaimRequest properly reclaims request in the queue', async () => {
+ const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);
+ const requestQueue = await RequestQueue.open();
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Mock reclaimRequest in requestQueue
+ const reclaimSpy = vi.spyOn(requestQueue, 'reclaimRequest');
+
+ // First fetch a request
+ const request = await tandem.fetchNextRequest();
+ expect(request).not.toBeNull();
+
+ // Reclaim the request
+ await tandem.reclaimRequest(request!);
+
+ // Verify the queue's reclaimRequest was called
+ expect(reclaimSpy).toHaveBeenCalledWith(request, undefined);
+ });
+
+ test('handledCount returns the queue handledCount', async () => {
+ const requestList = await RequestList.open(null, [
+ { url: 'https://example.com/1' },
+ { url: 'https://example.com/2' },
+ ]);
+ const requestQueue = await RequestQueue.open();
+
+ // Mock handledCount methods to return fixed values
+ vi.spyOn(requestList, 'handledCount').mockReturnValue(3);
+ vi.spyOn(requestQueue, 'handledCount').mockResolvedValue(2);
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Only the request queue counts
+ await expect(tandem.handledCount()).resolves.toBe(2);
+ });
+
+ test('isFinished returns true only when both list and queue are finished', async () => {
+ const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);
+ const requestQueue = await RequestQueue.open();
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Mock the isFinished methods
+ vi.spyOn(requestList, 'isFinished').mockResolvedValue(false);
+ vi.spyOn(requestQueue, 'isFinished').mockResolvedValue(false);
+
+ // Neither is finished, so tandem should not be finished
+ expect(await tandem.isFinished()).toBe(false);
+
+ // Only list is finished
+ vi.spyOn(requestList, 'isFinished').mockResolvedValue(true);
+ vi.spyOn(requestQueue, 'isFinished').mockResolvedValue(false);
+ expect(await tandem.isFinished()).toBe(false);
+
+ // Only queue is finished
+ vi.spyOn(requestList, 'isFinished').mockResolvedValue(false);
+ vi.spyOn(requestQueue, 'isFinished').mockResolvedValue(true);
+ expect(await tandem.isFinished()).toBe(false);
+
+ // Both are finished
+ vi.spyOn(requestList, 'isFinished').mockResolvedValue(true);
+ vi.spyOn(requestQueue, 'isFinished').mockResolvedValue(true);
+ expect(await tandem.isFinished()).toBe(true);
+ });
+
+ test('isEmpty returns true only when both list and queue are empty', async () => {
+ const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);
+ const requestQueue = await RequestQueue.open();
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Mock the isEmpty methods
+ vi.spyOn(requestList, 'isEmpty').mockResolvedValue(false);
+ vi.spyOn(requestQueue, 'isEmpty').mockResolvedValue(false);
+
+ // Neither is empty, so tandem should not be empty
+ expect(await tandem.isEmpty()).toBe(false);
+
+ // Only list is empty
+ vi.spyOn(requestList, 'isEmpty').mockResolvedValue(true);
+ vi.spyOn(requestQueue, 'isEmpty').mockResolvedValue(false);
+ expect(await tandem.isEmpty()).toBe(false);
+
+ // Only queue is empty
+ vi.spyOn(requestList, 'isEmpty').mockResolvedValue(false);
+ vi.spyOn(requestQueue, 'isEmpty').mockResolvedValue(true);
+ expect(await tandem.isEmpty()).toBe(false);
+
+ // Both are empty
+ vi.spyOn(requestList, 'isEmpty').mockResolvedValue(true);
+ vi.spyOn(requestQueue, 'isEmpty').mockResolvedValue(true);
+ expect(await tandem.isEmpty()).toBe(true);
+ });
+
+ test('handles failed batch transfer appropriately', async () => {
+ const requestList = await RequestList.open(null, [
+ { url: 'https://example.com/1' },
+ { url: 'https://example.com/2' },
+ ]);
+ const requestQueue = await RequestQueue.open();
+
+ // Mock the queue's addRequests to simulate failure
+ vi.spyOn(requestQueue, 'addRequest').mockRejectedValue(new Error('Batch add failed'));
+
+ // Mock the reclaimRequest method to verify it's called
+ const reclaimSpy = vi.spyOn(requestList, 'reclaimRequest');
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Attempt to fetch which should trigger the batch transfer
+ await tandem.fetchNextRequest();
+
+ // Verify that reclaimRequest was called to reclaim the failed requests
+ expect(reclaimSpy).toHaveBeenCalled();
+ });
+
+ test('added requests are forwarded to the underlying RequestQueue', async () => {
+ const requestList = await RequestList.open(null, []);
+ const requestQueue = await RequestQueue.open();
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Mock the addRequest method of the queue
+ const addRequestSpy = vi.spyOn(requestQueue, 'addRequestsBatched');
+
+ // Add a new request directly through tandem
+ const request = new Request({ url: 'https://example.com/new' });
+ await tandem.addRequestsBatched([request]);
+
+ // Verify the request was forwarded to the queue
+ expect(addRequestSpy).toHaveBeenCalledWith([request], undefined);
+ });
+
+ test('async iterator iterates through all requests', async () => {
+ const requestList = await RequestList.open(null, [
+ { url: 'https://example.com/1' },
+ { url: 'https://example.com/2' },
+ ]);
+
+ const requestQueue = await RequestQueue.open();
+ await requestQueue.addRequest({ url: 'https://example.com/3' });
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ // Iterate through all requests
+ const urls: string[] = [];
+ for await (const request of tandem) {
+ urls.push(request.url);
+ }
+
+ // Verify we got both URLs
+ expect(urls).toEqual(['https://example.com/1', 'https://example.com/2', 'https://example.com/3']);
+ });
+});