Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Puppeteer utils: Cache responses

Jaroslav Hejlekd96bcad0d018
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index f72a5ede8..7ec1ebb3e 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -197,6 +197,65 @@ const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'imag
});
};
+/**
+ * Enables caching of intercepted responses into provided object. Automaticaly enables request interception in puppeteer.
+ * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
+ * This issue should be resolved or atleast mitigated in future iterations of this feature.
+ * @param {Page} page Puppeteer's `Page` object
+ * @param {Object} cache Object in which responses are stored
+ * @param {Array<String|RegExp>} responseUrlRules List of rules that are used to check if response should be cached.
+ * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
+ * @return {Promise<void>}
+ * @memberof utils.puppeteer
+ */
+const cacheResponses = async (page, cache, responseUrlRules) => {
+ checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(cache, 'cache', 'Object');
+ checkParamOrThrow(responseUrlRules, 'responseUrlRules', 'Array');
+
+ // Required to be able to intercept requests
+ await page.setRequestInterception(true);
+
+ page.on('request', async (request) => {
+ const url = request.url();
+
+ if (cache[url]) {
+ await request.respond(cache[url]);
+ return;
+ }
+
+ request.continue();
+ });
+
+ page.on('response', async (response) => {
+ const url = response.url();
+
+ // Response is already cached, do nothing
+ if (cache[url]) return;
+
+ const shouldCache = responseUrlRules.some((rule) => {
+ if (typeof rule === 'string') return url.includes(rule);
+ if (rule instanceof RegExp) return rule.test(url);
+
+ // @TODO: Throw error?
+ return false;
+ });
+
+ try {
+ if (shouldCache) {
+ const buffer = await response.buffer();
+ cache[url] = {
+ url,
+ headers: response.headers(),
+ body: buffer,
+ };
+ }
+ } catch (e) {
+ // ignore errors, usualy means that buffer is empty or broken connection
+ }
+ });
+};
+
/**
* Compiles a Puppeteer script into an async function that may be executed at any time
* by providing it with the following object:
@@ -264,5 +323,6 @@ export const puppeteerUtils = {
enqueueRequestsFromClickableElements,
enqueueLinks,
blockResources,
+ cacheResponses,
compileScript,
};
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 9e8b96d27..df150712c 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -224,6 +224,38 @@ describe('Apify.utils.puppeteer', () => {
}
});
+ it('supports cacheResponses()', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true });
+ const cache = {};
+
+ const getResourcesLoadedFromWiki = async () => {
+ let downloadedBytes = 0;
+ const page = await browser.newPage();
+ // Cache all javascript files, png files and svg files
+ await Apify.utils.puppeteer.cacheResponses(page, cache, ['.js', /.+\.png/i, /.+\.svg/i]);
+ page.on('response', async (response) => {
+ if (cache[response.url()]) return;
+ try {
+ const buffer = await response.buffer();
+ downloadedBytes += buffer.byteLength;
+ } catch (e) {
+ // do nothing
+ }
+ });
+ await page.goto('https://www.wikipedia.org/', { waitUntil: 'networkidle0' });
+ await page.close();
+ return downloadedBytes;
+ };
+
+ try {
+ const bytesDownloadedOnFirstRun = await getResourcesLoadedFromWiki();
+ const bytesDownloadedOnSecondRun = await getResourcesLoadedFromWiki();
+ expect(bytesDownloadedOnSecondRun).to.be.below(bytesDownloadedOnFirstRun);
+ } finally {
+ await browser.close();
+ }
+ });
+
it('compileScript() works', async () => {
const { compileScript } = Apify.utils.puppeteer;
const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';