Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implement PR comments
Ondra Urban2dc02b8dae4b
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex 8928e8e26..f60d3daa4 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -15,6 +15,8 @@ const DEFAULT_OPTIONS = {log.error('CheerioCrawler: Request failed and reached maximum retries', details);},+ ignoreSslErrors: false,+ useApifyProxy: false,};/**@@ -190,11 +192,11 @@ class CheerioCrawler {checkParamOrThrow(requestOptions, 'options.requestOptions', 'Maybe Object');checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');- checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Maybe Boolean');- checkParamOrThrow(useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');- checkParamOrThrow(apifyProxyGroups, 'options.apifyProxyGroups', 'Maybe Array');+ checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Boolean');+ checkParamOrThrow(useApifyProxy, 'options.useApifyProxy', 'Boolean');+ checkParamOrThrow(apifyProxyGroups, 'options.apifyProxyGroups', 'Maybe [String]');checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');- checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');+ checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');// Enforce valid proxy configurationif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 6258623dd..25aa780e9 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -143,7 +143,33 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque* @ignore*/const enqueueLinksCache = new Map();-const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;+export const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;++/**+ * Helper factory used in the `enqeueLinks()` function.+ * @param {Array} pseudoUrls+ * @returns {Array}+ * @ignore+ */+export const constructPseudoUrlInstances = (pseudoUrls) => {+ return pseudoUrls.map((item, idx) => {+ // Get pseudoUrl instance from cache.+ let pUrl = enqueueLinksCache.get(item);+ if (pUrl) return pUrl;+ // Nothing in cache, make a new instance.+ checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'Object|String');+ if (item instanceof PseudoUrl) pUrl = item;+ else if (typeof item === 'string') pUrl = new PseudoUrl(item);+ else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));+ // Manage cache+ enqueueLinksCache.set(item, pUrl);+ if (enqueueLinksCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {+ const key = enqueueLinksCache.keys().next().value;+ enqueueLinksCache.delete(key);+ }+ return pUrl;+ });+};/*** Remove with 1.0.0@@ -195,23 +221,7 @@ const enqueueLinks = async (page, selector, requestQueue, pseudoUrls = []) => {checkParamOrThrow(pseudoUrls, 'pseudoUrls', 'Array');// Construct pseudoUrls from input where necessary.- const pseudoUrlInstances = pseudoUrls.map((item, idx) => {- // Get pseudoUrl instance from cache.- let pUrl = enqueueLinksCache.get(item);- if (pUrl) return pUrl;- // Nothing in cache, make a new instance.- checkParamOrThrow(item, `pseudoUrls[${idx}]`, 'Object|String');- if (item instanceof PseudoUrl) pUrl = item;- else if (typeof item === 'string') pUrl = new PseudoUrl(item);- else pUrl = new PseudoUrl(item.purl, _.omit(item, 'purl'));- // Manage cache- enqueueLinksCache.set(item, pUrl);- if (enqueueLinksCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {- const key = enqueueLinksCache.keys().next().value;- enqueueLinksCache.delete(key);- }- return pUrl;- });+ const pseudoUrlInstances = constructPseudoUrlInstances(pseudoUrls);/* istanbul ignore next */const getHrefs = linkEls => linkEls.map(link => link.href).filter(href => !!href);