Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Finalizing request middlewares and tests
Marek Trunkat26fb87623332
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 8be12ea51..8fde906d5 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -367,12 +367,14 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {if (method !== 'GET' || payload || !_.isEmpty(headers)) {const interceptRequestHandler = async (interceptedRequest) => {- if (method !== 'GET') interceptedRequest._method = method; // eslint-disable-line- if (payload) interceptedRequest._postData = payload; // eslint-disable-line- if (!_.isEmpty(headers)) interceptedRequest._headers = headers; // eslint-disable-line+ const overrides = {};+ if (method !== 'GET') overrides.method = method;+ if (payload) overrides.postData = payload;+ if (!_.isEmpty(headers)) overrides.headers = headers;++ await interceptedRequest.continue(overrides);await removeInterceptRequestHandler(page, interceptRequestHandler); // We wan't this to be called only for the initial request.- await interceptedRequest.continue();};await addInterceptRequestHandler(page, interceptRequestHandler);diff --git a/src/puppeteers_request_interception.js b/src/puppeteers_request_interception.jsindex 1ecff0ac7..7bebbd671 100644--- a/src/puppeteers_request_interception.js+++ b/src/puppeteers_request_interception.js@@ -16,10 +16,12 @@ const handleRequest = (request, interceptRequestHandlers) => {let wasAborted = false;let wasResponded = false;let wasContinued = false;+ const requestOverrides = {};const originalContinue = request.continue.bind(request);- request.continue = () => {+ request.continue = (overrides) => {wasContinued = true;+ Object.assign(requestOverrides, overrides);};request.abort = _.wrap(request.abort.bind(request), (abort, ...args) => {@@ -29,7 +31,7 @@ const handleRequest = (request, interceptRequestHandlers) => {});request.respond = _.wrap(request.respond.bind(request), (respond, ...args) => {- wasResponded = false;+ wasResponded = true;return respond(...args);});@@ -48,13 +50,53 @@ const handleRequest = (request, interceptRequestHandlers) => {return wasAborted || wasResponded;});- if (!wasAborted && !wasResponded) originalContinue();+ if (!wasAborted && !wasResponded) return originalContinue(requestOverrides);};/**- * Adds request interception handler in similar as `page.on('request', handler);` but in addition to that+ * Adds request interception handler in similar to `page.on('request', handler);` but in addition to that* supports multiple parallel handlers.*+ * All the handlers are executed in a serie as were added.+ * Each of the handlers must call one of `request.continue()`, `request.abort()` and `request.respond()`.+ * In addition to that any of the handlers may modify the request object by passing its overrides to `request.continue()`.+ * If multiple handlers modify same property then the last one wins.+ *+ * If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further+ * to any of the remaining handlers.+ *+ *+ * **Example usage:**+ *+ * ```javascript+ * // Replace images with placeholder.+ * await addInterceptRequestHandler(page, (request) => {+ * if (request.resourceType() === 'image') {+ * return request.respond({+ * statusCode: 200,+ * contentType: 'image/jpeg',+ * body: placeholderImageBuffer,+ * });+ * }+ * return request.continue();+ * });+ *+ * // Abort all the scripts.+ * await addInterceptRequestHandler(page, (request) => {+ * if (request.resourceType() === 'script') return request.abort();+ * return request.continue();+ * });+ *+ * // Change requests to post.+ * await addInterceptRequestHandler(page, (request) => {+ * return request.continue({+ * method: 'POST',+ * });+ * });+ *+ * await page.goto('http://example.com');+ * ```+ ** @param {Page} page* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.* @param {Function} middleware Request interception handler. See @TODO for more information.diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex cc52dc61a..455172de1 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -157,18 +157,16 @@ describe('Apify.utils.puppeteer', () => {const page = await browser.newPage();await Apify.utils.puppeteer.blockResources(page);page.on('response', response => loadedUrls.push(response.url()));- await page.setContent(`<html>- <body>- <link rel="stylesheet" type="text/css" href="https://example.com/style.css">- <img src="https://example.com/image.png" />- <script src="https://example.com/script.js" defer="defer">></script>- </body>- </html>`, { waitUntil: 'networkidle0' });+ await page.setContent(`<html><body>+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">+ <img src="https://example.com/image.png" />+ <script src="https://example.com/script.js" defer="defer">></script>+ </body></html>`, { waitUntil: 'networkidle0' });} finally {await browser.close();}- expect(loadedUrls).to.be.eql([+ expect(loadedUrls).to.have.members(['https://example.com/script.js',]);});@@ -181,18 +179,16 @@ describe('Apify.utils.puppeteer', () => {const page = await browser.newPage();await Apify.utils.puppeteer.blockResources(page, ['script']);page.on('response', response => loadedUrls.push(response.url()));- await page.setContent(`<html>- <body>- <link rel="stylesheet" type="text/css" href="https://example.com/style.css">- <img src="https://example.com/image.png" />- <script src="https://example.com/script.js" defer="defer">></script>- </body>- </html>`, { waitUntil: 'networkidle0' });+ await page.setContent(`<html><body>+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">+ <img src="https://example.com/image.png" />+ <script src="https://example.com/script.js" defer="defer">></script>+ </body></html>`, { waitUntil: 'networkidle0' });} finally {await browser.close();}- expect(loadedUrls).to.be.eql([+ expect(loadedUrls).to.have.members(['https://example.com/style.css','https://example.com/image.png',]);@@ -280,4 +276,29 @@ describe('Apify.utils.puppeteer', () => {browser.close();}});++ it('gotoExtended() works', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ try {+ const page = await browser.newPage();+ const request = new Apify.Request({+ url: 'https://api.apify.com/v2/browser-info',+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json; charset=utf-8',+ },+ payload: '{ "foo": "bar" }',+ });++ const response = await Apify.utils.puppeteer.gotoExtended(page, request);++ const { method, headers, bodyLength } = JSON.parse(await response.text());+ expect(method).to.be.eql('POST');+ expect(bodyLength).to.be.eql(16);+ expect(headers['content-type']).to.be.eql('application/json; charset=utf-8');+ } finally {+ await browser.close();+ }+ });});diff --git a/test/puppeteers_request_interception.js b/test/puppeteers_request_interception.jsnew file mode 100644index 000000000..cc9ac6711--- /dev/null+++ b/test/puppeteers_request_interception.js@@ -0,0 +1,191 @@+import { expect } from 'chai';+import Apify from '../build/index';++const { utils: { log } } = Apify;+const { addInterceptRequestHandler, removeInterceptRequestHandler } = Apify.utils.puppeteer;++// Simple page with image, script and stylesheet links.+const HTML_PAGE = `<html><body>+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">+ <img src="https://example.com/image.png" />+ <script src="https://example.com/script.js" defer="defer">></script>+</body></html>`;++describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandler()', () => {+ it('should allow multiple handlers', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ const allUrls = [];+ const loadedUrls = [];++ try {+ const page = await browser.newPage();++ // Just collect all URLs.+ await addInterceptRequestHandler(page, (request) => {+ allUrls.push(request.url());+ return request.continue();+ });++ // Abort images.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'image') return request.abort();+ return request.continue();+ });++ // Abort scripts.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'script') return request.abort();+ return request.continue();+ });++ // Save all loaded URLs.+ page.on('response', response => loadedUrls.push(response.url()));++ await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ } finally {+ await browser.close();+ }++ expect(allUrls).to.have.members([+ 'https://example.com/script.js',+ 'https://example.com/style.css',+ 'https://example.com/image.png',+ ]);++ expect(loadedUrls).to.have.members([+ 'https://example.com/style.css',+ ]);+ });++ it('should not propagate aborted/responded requests to following handlers', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });+ const propagatedUrls = [];++ try {+ const page = await browser.newPage();++ // Abort images.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'image') return request.abort();+ return request.continue();+ });++ // Respond scripts.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'script') {+ return request.respond({+ status: 404,+ });+ }+ return request.continue();+ });++ // Just collect all URLs propagated to the last handler.+ await addInterceptRequestHandler(page, (request) => {+ propagatedUrls.push(request.url());+ return request.continue();+ });+ await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ } finally {+ await browser.close();+ }++ expect(propagatedUrls).to.have.members([+ 'https://example.com/style.css',+ ]);+ });++ it('should allow to modify request', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ try {+ const page = await browser.newPage();++ // Change all requests to DELETE.+ await addInterceptRequestHandler(page, (request) => {+ return request.continue({ method: 'DELETE' });+ });+ // Add some query parameters to request URLs.+ await addInterceptRequestHandler(page, (request) => {+ return request.continue({+ headers: {+ 'Content-Type': 'application/json; charset=utf-8',+ },+ postData: '{ "foo": "bar" }',+ });+ });+ // Change all requests to POST and add payload.+ await addInterceptRequestHandler(page, (request) => {+ return request.continue({+ method: 'POST',+ });+ });++ // Check response that it's correct.+ const response = await page.goto('https://api.apify.com/v2/browser-info', { waitUntil: 'networkidle0' });+ const { method, headers, bodyLength } = JSON.parse(await response.text());+ expect(method).to.be.eql('POST');+ expect(bodyLength).to.be.eql(16);+ expect(headers['content-type']).to.be.eql('application/json; charset=utf-8');+ } finally {+ await browser.close();+ }+ });+});++describe('Apify.utils.puppeteer.removeInterceptRequestHandler()', () => {+ it('works', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ const loadedUrls = [];++ try {+ const page = await browser.newPage();+ page.on('response', response => loadedUrls.push(response.url()));++ // Abort images.+ const abortImagesHandler = (request) => {+ if (request.resourceType() === 'image') return request.abort();+ return request.continue();+ };+ await addInterceptRequestHandler(page, abortImagesHandler);++ // Abort scripts.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'script') return request.abort();+ return request.continue();+ });++ // Load with scripts and images disabled.+ await page.setContent('<html><body></body></html>');+ await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ expect(loadedUrls).to.have.members([+ 'https://example.com/style.css',+ ]);++ // Try it once again.+ await page.setContent('<html><body></body></html>');+ await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ expect(loadedUrls).to.have.members([+ 'https://example.com/style.css',+ 'https://example.com/style.css',+ ]);++ // Enable images.+ await removeInterceptRequestHandler(page, abortImagesHandler);++ // Try to load once again if images appear there.+ await page.setContent('<html><body></body></html>');+ await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ expect(loadedUrls).to.have.members([+ 'https://example.com/style.css',+ 'https://example.com/style.css',+ 'https://example.com/style.css',+ 'https://example.com/image.png',+ ]);+ } finally {+ await browser.close();+ }+ });+});