Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed docs for utils.downloadListOfUrls()
Jan Curn7e36d02b67db
diff --git a/src/utils.js b/src/utils.jsindex f8a463898..65572a567 100644--- a/src/utils.js+++ b/src/utils.js@@ -358,28 +358,26 @@ const sleep = (millis) => {* Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.* Optionally, custom regular expression and encoding may be provided.*- * @param {String} url- * @param {String} [encoding='utf8']- * @param {RegExp} [urlRegExp=URL_NO_COMMAS_REGEX]+ * @param {Object} options+ * @param {String} options.url+ * @param {String} [options.encoding='utf8']+ * @param {RegExp} [options.urlRegExp=URL_NO_COMMAS_REGEX]* @returns {Promise<String[]>}* @memberOf utils*/-const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX }) => {- try {- checkParamOrThrow(url, 'url', 'String');- checkParamOrThrow(encoding, 'string', 'String');- checkParamOrThrow(urlRegExp, 'urlRegExp', 'RegExp');- } catch (err) {- return Promise.reject(err);- }- return requestPromise.get({ url, encoding })- .then(string => extractUrls({ string, urlRegExp }));+const downloadListOfUrls = async ({ url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX }) => {+ checkParamOrThrow(url, 'url', 'String');+ checkParamOrThrow(encoding, 'string', 'String');+ checkParamOrThrow(urlRegExp, 'urlRegExp', 'RegExp');++ const string = await requestPromise.get({ url, encoding });+ return extractUrls({ string, urlRegExp });};/*** Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.* @param {String} string- * @param {RegExp} [urlRegExp=URL_NO_COMMAS_REGEX]+ * @param {RegExp} [urlRegExp=Apify.utils.URL_NO_COMMAS_REGEX]* @returns {String[]}* @memberOf utils*/