Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: requestList accepts proxyConfiguration for `requestsFromUrls` (#1317)
Jindřich Bärc120837e1737
diff --git a/src/request_list.js b/src/request_list.jsindex f83ad70c3..e5a31fb3f 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -20,7 +20,7 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';/*** @typedef RequestListOptions- * @property {Array<RequestOptions | Request | { requestsFromUrl: string } | string>} [sources]+ * @property {Array<RequestOptions | Request | { requestsFromUrl: string, regex?: RegExp } | string>} [sources]* An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,* plain objects that define at least the `url` property, or an array of {@link Request} instances.*@@ -91,6 +91,10 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';* // call the sourcesFunction again, saving time and resources.* await requestList.initialize();* ```+ * @property {ProxyConfiguration} [proxyConfiguration]+ * Used to pass the the proxy configuration for the `requestsFromUrls` objects.+ * Takes advantage of the internal address rotation and authentication process.+ * If undefined, the `requestsFromUrls` requests will be made without proxy.* @property {string} [persistStateKey]* Identifies the key in the default key-value store under which `RequestList` periodically stores its* state (i.e. which URLs were crawled and which not).@@ -219,6 +223,7 @@ export class RequestList {persistStateKey,persistRequestsKey,state,+ proxyConfiguration,keepDuplicateUrls = false,} = options;@@ -236,6 +241,7 @@ export class RequestList {inProgress: ow.object,}),keepDuplicateUrls: ow.optional.boolean,+ proxyConfiguration: ow.optional.object,}));this.log = log.child({ prefix: 'RequestList' });@@ -278,6 +284,9 @@ export class RequestList {// Will be empty after initialization to save memory.this.sources = sources || [];this.sourcesFunction = sourcesFunction;++ // The proxy configuration used for `requestsFromUrls` requests.+ this.proxyConfiguration = proxyConfiguration;}/**@@ -656,7 +665,7 @@ export class RequestList {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex });+ urlsArr = await downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex, proxyUrl: this.proxyConfiguration?.newUrl() });} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);}@@ -812,7 +821,7 @@ export class RequestList {** If `null`, the list will not be persisted and will only be stored in memory. Process restart* will then cause the list to be crawled again from the beginning. We suggest always using a name.- * @param {Array<RequestOptions|Request|string>} sources+ * @param {RequestListOptions['sources']} sources* An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,* plain objects that define at least the `url` property, or an array of {@link Request} instances.*diff --git a/src/utils.js b/src/utils.jsindex 48b67b85d..362db4581 100644--- a/src/utils.js+++ b/src/utils.js@@ -423,6 +423,7 @@ export const sleep = (millis) => {* @param {object} options* @param {string} options.url URL to the file* @param {string} [options.encoding='utf8'] The encoding of the file.+ * @param {string} [options.proxyUrl] The proxy url to be used for the request.* @param {RegExp} [options.urlRegExp=URL_NO_COMMAS_REGEX]* Custom regular expression to identify the URLs in the file to extract.* The regular expression should be case-insensitive and have global flag set (i.e. `/something/gi`).@@ -434,8 +435,9 @@ const downloadListOfUrls = async (options) => {url: ow.string.url,encoding: ow.optional.string,urlRegExp: ow.optional.regExp,+ proxyUrl: ow.optional.string,}));- const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX } = options;+ const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl } = options;// Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);@@ -445,7 +447,7 @@ const downloadListOfUrls = async (options) => {fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;}- const { body: string } = await requestAsBrowser({ url: fixedUrl, encoding });+ const { body: string } = await requestAsBrowser({ url: fixedUrl, encoding, proxyUrl });return extractUrls({ string, urlRegExp });};diff --git a/test/request_list.test.js b/test/request_list.test.jsindex 5483cb75a..65079dcd3 100644--- a/test/request_list.test.js+++ b/test/request_list.test.js@@ -133,12 +133,12 @@ describe('Apify.RequestList', () => {mock.expects('downloadListOfUrls').once()- .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })+ .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined, proxyUrl: undefined }).returns(new Promise((resolve) => setTimeout(resolve(list1), 100)));mock.expects('downloadListOfUrls').once()- .withArgs({ url: 'http://example.com/list-2', urlRegExp: undefined })+ .withArgs({ url: 'http://example.com/list-2', urlRegExp: undefined, proxyUrl: undefined }).returns(Promise.resolve(list2), 0);const requestList = new Apify.RequestList({@@ -168,7 +168,7 @@ describe('Apify.RequestList', () => {mock.expects('requestAsBrowser').once()- .withArgs({ url: 'http://example.com/list-1', encoding: 'utf8' })+ .withArgs({ url: 'http://example.com/list-1', encoding: 'utf8', proxyUrl: undefined }).resolves({ body: listStr });const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g;@@ -210,7 +210,7 @@ describe('Apify.RequestList', () => {mock.expects('requestAsBrowser').exactly(wrongUrls.length)- .withArgs({ url: correctUrl, encoding: 'utf8' })+ .withArgs({ url: correctUrl, encoding: 'utf8', proxyUrl: undefined }).resolves({ body: JSON.stringify(list) });const requestList = new Apify.RequestList({@@ -231,7 +231,7 @@ describe('Apify.RequestList', () => {const mock = sinon.mock(utils.publicUtils);mock.expects('downloadListOfUrls').once()- .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })+ .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined, proxyUrl: undefined }).returns(Promise.resolve([]));const requestList = new Apify.RequestList({@@ -251,6 +251,37 @@ describe('Apify.RequestList', () => {mock.restore();});+ test('should use the defined proxy server when using `requestsFromUrl`', async () => {+ const proxyUrls = [+ 'http://proxyurl.usedforthe.download',+ 'http://another.proxy.url',+ ];++ const mock = sinon.mock(utils.publicUtils);+ mock.expects('downloadListOfUrls')+ .thrice()+ .withArgs({ url: sinon.match(/https?:\/\/example\.com\/list-\d/), urlRegExp: undefined, proxyUrl: sinon.match.in(proxyUrls) })+ .resolves(['https://example.com/result']);++ const proxyConfiguration = await Apify.createProxyConfiguration({+ proxyUrls,+ });++ const requestList = new Apify.RequestList({+ sources: [+ { requestsFromUrl: 'http://example.com/list-1' },+ { requestsFromUrl: 'http://example.com/list-2' },+ { requestsFromUrl: 'http://example.com/list-3' },+ ],+ proxyConfiguration,+ });++ await requestList.initialize();++ mock.verify();+ mock.restore();+ });+test('should correctly handle reclaimed pages', async () => {const requestList = new Apify.RequestList({sources: [@@ -618,7 +649,7 @@ describe('Apify.RequestList', () => {// Expect downloadListOfUrls returns list of URLspublicUtilsMock.expects('downloadListOfUrls').once()- .withArgs({ url: 'http://example.com/list-urls.txt', urlRegExp: undefined })+ .withArgs({ url: 'http://example.com/list-urls.txt', urlRegExp: undefined, proxyUrl: undefined }).returns(Promise.resolve(urlsFromTxt));await requestList.initialize();