Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

cacheResponses function small improvements

Jaroslav Hejlekfd6b878d0af5
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 7ec1ebb3e..7bed9a760 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -213,6 +213,9 @@ const cacheResponses = async (page, cache, responseUrlRules) => {
checkParamOrThrow(cache, 'cache', 'Object');
checkParamOrThrow(responseUrlRules, 'responseUrlRules', 'Array');
+ // Check that rules are either String or RegExp
+ responseUrlRules.forEach((rule, index) => checkParamOrThrow(rule, `responseUrlRules[${index}]`, 'String | RegExp'));
+
// Required to be able to intercept requests
await page.setRequestInterception(true);
@@ -236,16 +239,13 @@ const cacheResponses = async (page, cache, responseUrlRules) => {
const shouldCache = responseUrlRules.some((rule) => {
if (typeof rule === 'string') return url.includes(rule);
if (rule instanceof RegExp) return rule.test(url);
-
- // @TODO: Throw error?
- return false;
});
try {
if (shouldCache) {
const buffer = await response.buffer();
cache[url] = {
- url,
+ status: response.status(),
headers: response.headers(),
body: buffer,
};
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index df150712c..636d66a49 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -256,6 +256,30 @@ describe('Apify.utils.puppeteer', () => {
}
});
+ it('cacheResponses() throws when rule with invalid type is provided', async () => {
+ const mockedPage = {
+ setRequestInterception: () => {},
+ on: () => {},
+ };
+
+ const testRuleType = async (value) => {
+ try {
+ await Apify.utils.puppeteer.cacheResponses(mockedPage, {}, [value]);
+ } catch (error) {
+ // this is valid path for this test
+ return;
+ }
+
+ expect(`Rule '${value}' should have thrown error`).to.be.equal('');
+ };
+ await testRuleType(0);
+ await testRuleType(1);
+ await testRuleType(null);
+ await testRuleType([]);
+ await testRuleType(['']);
+ await testRuleType(() => {});
+ });
+
it('compileScript() works', async () => {
const { compileScript } = Apify.utils.puppeteer;
const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';