Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add support for `requestsFromUrl` to `RequestQueue` (#1917)
Martin Adámek7f2557cdbbde
diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 9b0814d4a..54f8fdacd 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -440,3 +440,11 @@ interface ComputeUniqueKeyOptions {keepUrlFragment?: boolean;useExtendedUniqueKey?: boolean;}++export type Source = (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | Request;++/** @internal */+export interface InternalSource {+ requestsFromUrl: string;+ regex?: RegExp;+}diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 4b690ad0d..a978afb4a 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -5,7 +5,7 @@ import type { EventManager } from '../events';import { EventType } from '../events';import { Configuration } from '../configuration';import { log } from '../log';-import type { RequestOptions } from '../request';+import type { InternalSource, RequestOptions, Source } from '../request';import { Request } from '../request';import { createDeserialize, serializeArray } from '../serialization';import { KeyValueStore } from './key_value_store';@@ -54,7 +54,7 @@ export interface RequestListOptions {* ]* ```*/- sources?: Source[];+ sources?: RequestListSource[];/*** A function that will be called to get the sources for the `RequestList`, but only if `RequestList`@@ -97,9 +97,9 @@ export interface RequestListOptions {sourcesFunction?: RequestListSourcesFunction;/**- * Used to pass the the proxy configuration for the `requestsFromUrls` objects.- * Takes advantage of the internal address rotation and authentication process.- * If undefined, the `requestsFromUrls` requests will be made without proxy.+ * Used to pass the proxy configuration for the `requestsFromUrl` objects.+ * Takes advantage of the internal address rotation and authentication process.+ * If undefined, the `requestsFromUrl` requests will be made without proxy.*/proxyConfiguration?: ProxyConfiguration;@@ -279,7 +279,7 @@ export class RequestList {private initialState?: RequestListState;private store?: KeyValueStore;private keepDuplicateUrls: boolean;- private sources: Source[];+ private sources: RequestListSource[];private sourcesFunction?: RequestListSourcesFunction;private proxyConfiguration?: ProxyConfiguration;private events: EventManager;@@ -330,7 +330,7 @@ export class RequestList {this.sources = sources ? [...sources] : [];this.sourcesFunction = sourcesFunction;- // The proxy configuration used for `requestsFromUrls` requests.+ // The proxy configuration used for `requestsFromUrl` requests.this.proxyConfiguration = proxyConfiguration;}@@ -695,7 +695,7 @@ export class RequestList {* If the `source` parameter is a string or plain object and not an instance* of a `Request`, then the function creates a `Request` instance.*/- protected _addRequest(source: Source) {+ protected _addRequest(source: RequestListSource) {let request;const type = typeof source;if (type === 'string') {@@ -829,7 +829,11 @@ export class RequestList {* the {@apilink RequestListOptions.persistStateKey} and {@apilink RequestListOptions.persistRequestsKey}* options and the `sources` parameter supersedes the {@apilink RequestListOptions.sources} option.*/- static async open(listNameOrOptions: string | null | RequestListOptions, sources?: Source[], options: RequestListOptions = {}): Promise<RequestList> {+ static async open(+ listNameOrOptions: string | null | RequestListOptions,+ sources?: RequestListSource[],+ options: RequestListOptions = {},+ ): Promise<RequestList> {if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {options = { ...listNameOrOptions, ...options };const rl = new RequestList(options);@@ -892,6 +896,5 @@ export interface RequestListState {}-export type Source = string | (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | Request;-interface InternalSource { requestsFromUrl: string; regex?: RegExp }-export type RequestListSourcesFunction = () => Promise<Source[]>;+type RequestListSource = string | Source;+export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 101df1fa6..607b612be 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -12,13 +12,15 @@ import type {RequestQueueInfo,StorageClient,} from '@crawlee/types';+import { downloadListOfUrls } from '@crawlee/utils';import type { StorageManagerOptions } from './storage_manager';import { StorageManager } from './storage_manager';import { log } from '../log';-import type { RequestOptions } from '../request';+import type { InternalSource, RequestOptions, Source } from '../request';import { Request } from '../request';import { Configuration } from '../configuration';import { purgeDefaultStorages } from './utils';+import type { ProxyConfiguration } from '../proxy_configuration';const MAX_CACHED_REQUESTS = 1_000_000;@@ -160,6 +162,7 @@ export class RequestQueue {timeoutSecs = 30;clientKey = cryptoRandomObjectId();client: RequestQueueClient;+ private proxyConfiguration?: ProxyConfiguration;/*** Contains a cached list of request IDs from the head of the queue,@@ -211,6 +214,7 @@ export class RequestQueue {clientKey: this.clientKey,timeoutSecs: this.timeoutSecs,}) as RequestQueueClient;+ this.proxyConfiguration = options.proxyConfiguration;}/**@@ -234,11 +238,8 @@ export class RequestQueue {* Note that the function sets the `uniqueKey` and `id` fields to the passed Request.* @param [options] Request queue operation options.*/- async addRequest(requestLike: Request | RequestOptions, options: RequestQueueOperationOptions = {}): Promise<RequestQueueOperationInfo> {- ow(requestLike, ow.object.partialShape({- url: ow.string,- id: ow.undefined,- }));+ async addRequest(requestLike: Source, options: RequestQueueOperationOptions = {}): Promise<RequestQueueOperationInfo> {+ ow(requestLike, ow.object);ow(options, ow.object.exactShape({forefront: ow.optional.boolean,}));@@ -246,6 +247,18 @@ export class RequestQueue {this.lastActivity = new Date();const { forefront = false } = options;+ if ('requestsFromUrl' in requestLike) {+ const requests = await this._fetchRequestsFromUrl(requestLike as InternalSource);+ const processedRequests = await this._addFetchedRequests(requestLike as InternalSource, requests, options);++ return processedRequests[0];+ }++ ow(requestLike, ow.object.partialShape({+ url: ow.string,+ id: ow.undefined,+ }));+const request = requestLike instanceof Request? requestLike: new Request(requestLike);@@ -293,13 +306,10 @@ export class RequestQueue {* @param [options] Request queue operation options.*/async addRequests(- requestsLike: (Request | RequestOptions)[],+ requestsLike: Source[],options: RequestQueueOperationOptions = {},): Promise<BatchAddRequestsResult> {- ow(requestsLike, ow.array.ofType(ow.object.partialShape({- url: ow.string,- id: ow.undefined,- })));+ ow(requestsLike, ow.array);ow(options, ow.object.exactShape({forefront: ow.optional.boolean,}));@@ -323,11 +333,18 @@ export class RequestQueue {unprocessedRequests: [],};- const requests = requestsLike.map((requestLike) => {- return requestLike instanceof Request- ? requestLike- : new Request(requestLike);- });+ for (const requestLike of requestsLike) {+ if ('requestsFromUrl' in requestLike) {+ const requests = await this._fetchRequestsFromUrl(requestLike as InternalSource);+ await this._addFetchedRequests(requestLike as InternalSource, requests, options);+ }+ }++ const requests = requestsLike+ .filter((requestLike) => !('requestsFromUrl' in requestLike))+ .map((requestLike) => {+ return requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions);+ });const requestsToAdd = new Map<string, Request>();@@ -768,6 +785,55 @@ export class RequestQueue {return this.client.get();}+ /**+ * Fetches URLs from requestsFromUrl and returns them in format of list of requests+ */+ protected async _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {+ const { requestsFromUrl, regex, ...sharedOpts } = source;++ // Download remote resource and parse URLs.+ let urlsArr;+ try {+ urlsArr = await this._downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex, proxyUrl: await this.proxyConfiguration?.newUrl() });+ } catch (err) {+ throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);+ }++ // Skip if resource contained no URLs.+ if (!urlsArr.length) {+ this.log.warning('list fetched, but it is empty.', { requestsFromUrl, regex });+ return [];+ }++ return urlsArr.map((url) => ({ url, ...sharedOpts }));+ }++ /**+ * Adds all fetched requests from a URL from a remote resource.+ */+ protected async _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions) {+ const { requestsFromUrl, regex } = source;+ const { processedRequests } = await this.addRequests(fetchedRequests, options);++ this.log.info('Fetched and loaded Requests from a remote resource.', {+ requestsFromUrl,+ regex,+ fetchedCount: fetchedRequests.length,+ importedCount: processedRequests.length,+ duplicateCount: fetchedRequests.length - processedRequests.length,+ sample: JSON.stringify(fetchedRequests.slice(0, 5)),+ });++ return processedRequests;+ }++ /**+ * @internal wraps public utility for mocking purposes+ */+ private async _downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<string[]> {+ return downloadListOfUrls(options);+ }+/*** Opens a request queue and returns a promise resolving to an instance* of the {@apilink RequestQueue} class.@@ -785,10 +851,11 @@ export class RequestQueue {* @param [options] Open Request Queue options.*/static async open(queueIdOrName?: string | null, options: StorageManagerOptions = {}): Promise<RequestQueue> {- ow(queueIdOrName, ow.optional.string);+ ow(queueIdOrName, ow.optional.any(ow.string, ow.null));ow(options, ow.object.exactShape({config: ow.optional.object.instanceOf(Configuration),storageClient: ow.optional.object,+ proxyConfiguration: ow.optional.object,}));options.config ??= Configuration.getGlobalConfig();@@ -797,8 +864,10 @@ export class RequestQueue {await purgeDefaultStorages(options.config, options.storageClient);const manager = StorageManager.getManager(this, options.config);+ const queue = await manager.openStorage(queueIdOrName, options.storageClient);+ queue.proxyConfiguration = options.proxyConfiguration;- return manager.openStorage(queueIdOrName, options.storageClient);+ return queue;}}@@ -806,4 +875,11 @@ export interface RequestQueueOptions {id: string;name?: string;client: StorageClient;++ /**+ * Used to pass the proxy configuration for the `requestsFromUrl` objects.+ * Takes advantage of the internal address rotation and authentication process.+ * If undefined, the `requestsFromUrl` requests will be made without proxy.+ */+ proxyConfiguration?: ProxyConfiguration;}diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.tsindex 6d25a6cdf..54ad10098 100644--- a/packages/core/src/storages/storage_manager.ts+++ b/packages/core/src/storages/storage_manager.ts@@ -2,6 +2,7 @@ import type { Dictionary, StorageClient } from '@crawlee/types';import { AsyncQueue } from '@sapphire/async-queue';import { Configuration } from '../configuration';import type { Constructor } from '../typedefs';+import type { ProxyConfiguration } from '../proxy_configuration';const DEFAULT_ID_CONFIG_KEYS = {Dataset: 'defaultDatasetId',@@ -156,4 +157,11 @@ export interface StorageManagerOptions {* Optional storage client that should be used to open storages.*/storageClient?: StorageClient;++ /**+ * Used to pass the proxy configuration for the `requestsFromUrl` objects.+ * Takes advantage of the internal address rotation and authentication process.+ * If undefined, the `requestsFromUrl` requests will be made without proxy.+ */+ proxyConfiguration?: ProxyConfiguration;}diff --git a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.tsindex 97c0ed5c0..60dcf1e1c 100644--- a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts+++ b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts@@ -1,6 +1,6 @@import log from '@apify/log';import { load } from 'cheerio';-import type { CheerioRoot, Request, RequestOptions } from 'crawlee';+import type { CheerioRoot, Source } from 'crawlee';import { EnqueueStrategy, Configuration, cheerioCrawlerEnqueueLinks, RequestQueue } from 'crawlee';const apifyClient = Configuration.getStorageClient();@@ -20,7 +20,7 @@ const HTML = ``;function getMockRequestQueue() {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });diff --git a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.tsindex e5f933f68..9c4fba7d5 100644--- a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts+++ b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts@@ -1,6 +1,6 @@import log from '@apify/log';import { load } from 'cheerio';-import type { CheerioRoot, Request, RequestOptions } from 'crawlee';+import type { CheerioRoot, Source } from 'crawlee';import { Configuration, cheerioCrawlerEnqueueLinks, RequestQueue, EnqueueStrategy } from 'crawlee';const apifyClient = Configuration.getStorageClient();@@ -32,7 +32,7 @@ const HTML = ``;function getMockRequestQueue() {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });diff --git a/packages/core/test/enqueue_links/userData.test.ts b/packages/core/test/enqueue_links/userData.test.tsindex 2a6fe45b9..0d04ea80c 100644--- a/packages/core/test/enqueue_links/userData.test.ts+++ b/packages/core/test/enqueue_links/userData.test.ts@@ -1,7 +1,8 @@import log from '@apify/log';+import type { CheerioAPI } from 'cheerio';import { load } from 'cheerio';-import type { CheerioRoot, Request, RequestOptions } from 'crawlee';-import { Configuration, cheerioCrawlerEnqueueLinks, RequestQueue, EnqueueStrategy } from 'crawlee';+import type { Source } from '@crawlee/cheerio';+import { Configuration, cheerioCrawlerEnqueueLinks, RequestQueue } from '@crawlee/cheerio';const apifyClient = Configuration.getStorageClient();@@ -20,7 +21,7 @@ const HTML = ``;function getMockRequestQueue() {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });@@ -43,7 +44,7 @@ describe('enqueueLinks() - userData shouldn\'t be changed and outer label must tlog.setLevel(ll);});- let $: CheerioRoot;+ let $: CheerioAPI;beforeEach(() => {$ = load(HTML);});diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.tsindex 209f16a8a..fbe76218f 100644--- a/test/core/enqueue_links/click_elements.test.ts+++ b/test/core/enqueue_links/click_elements.test.ts@@ -1,4 +1,4 @@-import type { Request, RequestOptions, RequestQueueOperationOptions } from 'crawlee';+import type { Request, RequestQueueOperationOptions, Source } from 'crawlee';import {Configuration,RequestQueue,@@ -72,7 +72,7 @@ testCases.forEach(({});test('should work', async () => {- const addedRequests: (Request | RequestOptions)[] = [];+ const addedRequests: Source[] = [];const requestQueue = Object.create(RequestQueue.prototype);requestQueue.addRequests = async (request: Request[]) => addedRequests.push(...request);const html = `@@ -102,7 +102,7 @@ testCases.forEach(({});test('accepts forefront option', async () => {- const addedRequests: {request: (Request | RequestOptions); options: RequestQueueOperationOptions}[] = [];+ const addedRequests: {request: Source; options: RequestQueueOperationOptions}[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });requestQueue.addRequests = async (requests, options) => {addedRequests.push(...requests.map((request) => ({ request, options })));diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex 5f2e0270d..fbee883d9 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -1,7 +1,7 @@import log from '@apify/log';import type { CheerioRoot } from '@crawlee/utils';-import cheerio from 'cheerio';-import type { Request, RequestOptions, RequestQueueOperationOptions } from '@crawlee/puppeteer';+import { load } from 'cheerio';+import type { RequestQueueOperationOptions, Source } from '@crawlee/puppeteer';import {browserCrawlerEnqueueLinks,Configuration,@@ -74,7 +74,7 @@ describe('enqueueLinks()', () => {});test('works with item limit', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -107,7 +107,7 @@ describe('enqueueLinks()', () => {});test('works with globs', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -152,7 +152,7 @@ describe('enqueueLinks()', () => {});test('works with regexps', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -196,7 +196,7 @@ describe('enqueueLinks()', () => {});test('works with exclude glob', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -243,7 +243,7 @@ describe('enqueueLinks()', () => {});test('works with exclude regexp', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -290,7 +290,7 @@ describe('enqueueLinks()', () => {});test('works with pseudoUrls', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -334,7 +334,7 @@ describe('enqueueLinks()', () => {});test('throws with RegExp pseudoUrls', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -357,7 +357,7 @@ describe('enqueueLinks()', () => {});test('works with undefined pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -392,7 +392,7 @@ describe('enqueueLinks()', () => {});test('throws with null pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -409,7 +409,7 @@ describe('enqueueLinks()', () => {});test('works with empty pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -444,7 +444,7 @@ describe('enqueueLinks()', () => {});test('throws with sparse pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -468,7 +468,7 @@ describe('enqueueLinks()', () => {});test('correctly resolves relative URLs with default strategy of same-hostname', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -495,7 +495,7 @@ describe('enqueueLinks()', () => {});test('correctly resolves relative URLs with the strategy of same-domain', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -526,7 +526,7 @@ describe('enqueueLinks()', () => {});test('correctly resolves relative URLs with the strategy of all', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -576,7 +576,7 @@ describe('enqueueLinks()', () => {});test('correctly works with transformRequestFunction', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -627,7 +627,7 @@ describe('enqueueLinks()', () => {let $: CheerioRoot;beforeEach(async () => {- $ = cheerio.load(HTML);+ $ = load(HTML);});afterEach(async () => {@@ -635,7 +635,7 @@ describe('enqueueLinks()', () => {});test('works with globs', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -679,7 +679,7 @@ describe('enqueueLinks()', () => {});test('works with RegExps', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -723,7 +723,7 @@ describe('enqueueLinks()', () => {});test('works with string pseudoUrls', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -767,7 +767,7 @@ describe('enqueueLinks()', () => {});test('throws with RegExp pseudoUrls', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -788,7 +788,7 @@ describe('enqueueLinks()', () => {});test('works with undefined pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -822,7 +822,7 @@ describe('enqueueLinks()', () => {});test('throws with null pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -838,7 +838,7 @@ describe('enqueueLinks()', () => {});test('works with empty pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -872,7 +872,7 @@ describe('enqueueLinks()', () => {});test('throws with sparse pseudoUrls[]', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -894,7 +894,7 @@ describe('enqueueLinks()', () => {});test('correctly resolves relative URLs with the strategy of all', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -944,7 +944,7 @@ describe('enqueueLinks()', () => {});test('correctly resolves relative URLs with the default strategy of same-hostname', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testingrequestQueue.addRequests = async (request) => {@@ -970,7 +970,7 @@ describe('enqueueLinks()', () => {});test('correctly resolves relative URLs with the strategy of same-domain', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -1001,7 +1001,7 @@ describe('enqueueLinks()', () => {});test('correctly works with transformRequestFunction', async () => {- const enqueued: (Request | RequestOptions)[] = [];+ const enqueued: Source[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });// @ts-expect-error Override method for testing@@ -1047,7 +1047,7 @@ describe('enqueueLinks()', () => {});test('accepts forefront option', async () => {- const enqueued: { request: (Request | RequestOptions); options: RequestQueueOperationOptions }[] = [];+ const enqueued: { request: Source; options: RequestQueueOperationOptions }[] = [];const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });requestQueue.addRequests = async (requests, options) => {diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex 2a3eddf6c..391ddee8d 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -5,8 +5,31 @@ import {RequestQueue,Request,Configuration,+ ProxyConfiguration,} from '@crawlee/core';import { sleep } from '@crawlee/utils';+import { gotScraping } from 'got-scraping';+import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';++jest.mock('got-scraping', () => {+ const original: typeof import('got-scraping') = jest.requireActual('got-scraping');+ return {+ ...original,+ gotScraping: jest.fn(original.gotScraping),+ };+});++const gotScrapingSpy = gotScraping as jest.MockedFunction<typeof gotScraping>;+const originalGotScraping = gotScrapingSpy.getMockImplementation()!;++afterEach(() => {+ gotScrapingSpy.mockReset();+ gotScrapingSpy.mockImplementation(originalGotScraping);+});++afterAll(() => {+ jest.unmock('got-scraping');+});describe('RequestQueue remote', () => {const storageClient = Configuration.getStorageClient();@@ -686,6 +709,144 @@ describe('RequestQueue remote', () => {});});+describe('RequestQueue with requestsFromUrl', () => {+ const emulator = new MemoryStorageEmulator();++ beforeEach(async () => {+ await emulator.init();+ jest.restoreAllMocks();+ });++ afterAll(async () => {+ await emulator.destroy();+ });++ test('should correctly load list from hosted files in correct order', async () => {+ const spy = jest.spyOn(RequestQueue.prototype as any, '_downloadListOfUrls');+ const list1 = [+ 'https://example.com',+ 'https://google.com',+ 'https://wired.com',+ ];+ const list2 = [+ 'https://another.com',+ 'https://page.com',+ ];+ spy.mockImplementationOnce(() => new Promise((resolve) => setTimeout(resolve(list1) as any, 100)) as any);+ spy.mockResolvedValueOnce(list2);++ const queue = await RequestQueue.open();+ await queue.addRequests([+ { method: 'GET', requestsFromUrl: 'http://example.com/list-1' },+ { method: 'POST', requestsFromUrl: 'http://example.com/list-2' },+ ]);++ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[0] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[1] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[2] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[0] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[1] });++ expect(spy).toBeCalledTimes(2);+ expect(spy).toBeCalledWith({ url: 'http://example.com/list-1', urlRegExp: undefined });+ expect(spy).toBeCalledWith({ url: 'http://example.com/list-2', urlRegExp: undefined });+ spy.mockRestore();+ });++ test('should use regex parameter to parse urls', async () => {+ const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd';+ const listArr = ['https://example.com', 'HTTP://google.com'];+ gotScrapingSpy.mockResolvedValue({ body: listStr } as any);++ const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g;+ const queue = await RequestQueue.open();+ await queue.addRequest({+ method: 'GET',+ requestsFromUrl: 'http://example.com/list-1',+ regex,+ });++ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[0] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[1] });+ await queue.drop();++ expect(gotScrapingSpy).toBeCalledWith({ url: 'http://example.com/list-1', encoding: 'utf8' });+ gotScrapingSpy.mockRestore();+ });++ test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => {+ const list = [+ 'https://example.com',+ 'https://google.com',+ 'https://wired.com',+ ];+ const wrongUrls = [+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/123123132',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/?q=blablabla',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit#gid=0',+ ];+ const correctUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv';++ gotScrapingSpy.mockResolvedValue({ body: JSON.stringify(list) } as any);++ const queue = await RequestQueue.open();+ await queue.addRequests(wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl })));++ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] });+ expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] });++ expect(gotScrapingSpy).toBeCalledWith({ url: correctUrl, encoding: 'utf8' });+ gotScrapingSpy.mockRestore();+ await queue.drop();+ });++ test('should handle requestsFromUrl with no URLs', async () => {+ const spy = jest.spyOn(RequestQueue.prototype as any, '_downloadListOfUrls');+ spy.mockResolvedValueOnce([]);++ const queue = await RequestQueue.open();+ await queue.addRequest({+ method: 'GET',+ requestsFromUrl: 'http://example.com/list-1',+ });++ expect(await queue.fetchNextRequest()).toBe(null);++ expect(spy).toBeCalledTimes(1);+ expect(spy).toBeCalledWith({ url: 'http://example.com/list-1', urlRegExp: undefined });+ spy.mockRestore();+ });++ test('should use the defined proxy server when using `requestsFromUrl`', async () => {+ const proxyUrls = [+ 'http://proxyurl.usedforthe.download',+ 'http://another.proxy.url',+ ];++ const spy = jest.spyOn(RequestQueue.prototype as any, '_downloadListOfUrls');+ spy.mockResolvedValue([]);++ const proxyConfiguration = new ProxyConfiguration({+ proxyUrls,+ });++ const queue = await RequestQueue.open(null, { proxyConfiguration });+ await queue.addRequests([+ { requestsFromUrl: 'http://example.com/list-1' },+ { requestsFromUrl: 'http://example.com/list-2' },+ { requestsFromUrl: 'http://example.com/list-3' },+ ]);++ expect(spy).not.toBeCalledWith(expect.not.objectContaining({ proxyUrl: expect.any(String) }));++ spy.mockRestore();+ });+});+describe('RequestQueue v2', () => {const totalRequestsPerTest = 100;