Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
cacheResponses function small improvements
Jaroslav Hejlekfd6b878d0af5
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 7ec1ebb3e..7bed9a760 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -213,6 +213,9 @@ const cacheResponses = async (page, cache, responseUrlRules) => {checkParamOrThrow(cache, 'cache', 'Object');checkParamOrThrow(responseUrlRules, 'responseUrlRules', 'Array');+ // Check that rules are either String or RegExp+ responseUrlRules.forEach((rule, index) => checkParamOrThrow(rule, `responseUrlRules[${index}]`, 'String | RegExp'));+// Required to be able to intercept requestsawait page.setRequestInterception(true);@@ -236,16 +239,13 @@ const cacheResponses = async (page, cache, responseUrlRules) => {const shouldCache = responseUrlRules.some((rule) => {if (typeof rule === 'string') return url.includes(rule);if (rule instanceof RegExp) return rule.test(url);-- // @TODO: Throw error?- return false;});try {if (shouldCache) {const buffer = await response.buffer();cache[url] = {- url,+ status: response.status(),headers: response.headers(),body: buffer,};diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex df150712c..636d66a49 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -256,6 +256,30 @@ describe('Apify.utils.puppeteer', () => {}});+ it('cacheResponses() throws when rule with invalid type is provided', async () => {+ const mockedPage = {+ setRequestInterception: () => {},+ on: () => {},+ };++ const testRuleType = async (value) => {+ try {+ await Apify.utils.puppeteer.cacheResponses(mockedPage, {}, [value]);+ } catch (error) {+ // this is valid path for this test+ return;+ }++ expect(`Rule '${value}' should have thrown error`).to.be.equal('');+ };+ await testRuleType(0);+ await testRuleType(1);+ await testRuleType(null);+ await testRuleType([]);+ await testRuleType(['']);+ await testRuleType(() => {});+ });+it('compileScript() works', async () => {const { compileScript } = Apify.utils.puppeteer;const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';