Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added async handler option to request interception. (#664)
petrpatekf4223736b8a4
diff --git a/package.json b/package.jsonindex 1aad71b31..2a0c7208e 100644--- a/package.json+++ b/package.json@@ -43,7 +43,7 @@"build": "npm run clean && tsc -p tsconfig.json && node ./tools/typescript_fixes.js","build-docs": "npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js",- "test": "npm run build && jest --testTimeout=20000 --maxWorkers=3",+ "test": "npm run build && jest --testTimeout=20000 --maxWorkers=3 --silent","prepare": "npm run build","prepublishOnly": "(test $TRAVIS || (echo \"Publishing is reserved to the Travis CI!\"; exit 1))","clean": "rm -rf ./build ./types",diff --git a/src/puppeteer_request_interception.js b/src/puppeteer_request_interception.jsindex 7ae9d0717..455d670b8 100644--- a/src/puppeteer_request_interception.js+++ b/src/puppeteer_request_interception.js@@ -1,3 +1,4 @@+import { EventEmitter } from 'events';import * as _ from 'underscore';import { checkParamOrThrow } from 'apify-client/build/utils';import { Request as PuppeteerRequest, Page } from 'puppeteer'; // eslint-disable-line no-unused-vars@@ -5,6 +6,35 @@ import { Request as PuppeteerRequest, Page } from 'puppeteer'; // eslint-disable// We use weak maps here so that the content gets discarted after page gets closed.const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.+const pageInterceptedRequestsMap = new WeakMap(); // Maps page to a set of its pending intercepted requests.++/**+ * Enables observation of changes of internal state+ * to be able to queue other actions based on it.+ * @ignore+ */+class ObservableSet extends EventEmitter {+ constructor() {+ super();+ this.set = new Set();+ }++ add(value) {+ this.set.add(value);+ this.emit('add', value);+ return this.set;+ }++ delete(value) {+ const success = this.set.delete(value);+ this.emit('delete', value);+ return success;+ }++ get size() {+ return this.set.size;+ }+}/*** @callback InterceptHandler@@ -18,7 +48,7 @@ const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to mast* @param {Array<InterceptHandler>} interceptRequestHandlers An array of intercept request handlers.* @ignore*/-const handleRequest = (request, interceptRequestHandlers) => {+const handleRequest = async (request, interceptRequestHandlers) => {// If there are no intercept handlers, it means that request interception is not enabled (anymore)// and therefore .abort() .respond() and .continue() would throw and crash the process.if (!interceptRequestHandlers.length) return;@@ -49,21 +79,20 @@ const handleRequest = (request, interceptRequestHandlers) => {return respond(...args);});- _.some(interceptRequestHandlers, (handler) => {+ for (const handler of interceptRequestHandlers) {wasContinued = false;- handler(request);-+ await handler(request);// Check that one of the functions was called.if (!wasAborted && !wasResponded && !wasContinued) {throw new Error('Intercept request handler must call one of request.continue|respond|abort() methods!');}// If request was aborted or responded then we can finish immediately.- return wasAborted || wasResponded;- });+ if (wasAborted || wasResponded) return;+ }- if (!wasAborted && !wasResponded) return originalContinue(accumulatedOverrides);+ return originalContinue(accumulatedOverrides);};/**@@ -123,10 +152,12 @@ export const addInterceptRequestHandler = async (page, handler) => {checkParamOrThrow(page, 'page', 'Object');checkParamOrThrow(handler, 'handler', 'Function');- // We haven't initiated an array of handlers yet.if (!pageInterceptRequestHandlersMap.has(page)) {pageInterceptRequestHandlersMap.set(page, []);}+ if (!pageInterceptedRequestsMap.has(page)) {+ pageInterceptedRequestsMap.set(page, new ObservableSet());+ }const handlersArray = pageInterceptRequestHandlersMap.get(page);handlersArray.push(handler);@@ -135,9 +166,18 @@ export const addInterceptRequestHandler = async (page, handler) => {if (handlersArray.length === 1) {await page.setRequestInterception(true);- // This is a handler that get's set in page.on('request', ...) and that executes all the user+ // This is a handler that gets set in page.on('request', ...) and that executes all the user// added custom handlers.- const masterHandler = request => handleRequest(request, pageInterceptRequestHandlersMap.get(page));+ const masterHandler = async (request) => {+ const interceptedRequests = pageInterceptedRequestsMap.get(page);+ interceptedRequests.add(request);+ const interceptHandlers = pageInterceptRequestHandlersMap.get(page);+ try {+ await handleRequest(request, interceptHandlers);+ } finally {+ interceptedRequests.delete(request);+ }+ };pageInterceptRequestMasterHandlerMap.set(page, masterHandler);page.on('request', masterHandler);@@ -161,10 +201,27 @@ export const removeInterceptRequestHandler = async (page, handler) => {pageInterceptRequestHandlersMap.set(page, handlersArray);- // There are no more handlers so we can turn off request interception and remove master handler.- if (handlersArray.length === 0) {- await page.setRequestInterception(false);- const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);- page.removeListener('request', requestHandler);+ if (handlersArray.size === 0) {+ const interceptedRequestsInProgress = pageInterceptedRequestsMap.get(page);+ // Since handlers can be async, we can't simply turn off request interception+ // when there are no handlers, because some handlers could still+ // be in progress and request.abort|respond|continue() would throw.+ if (interceptedRequestsInProgress.size === 0) {+ await disableRequestInterception(page);+ } else {+ const onDelete = () => {+ if (interceptedRequestsInProgress.size === 0) {+ disableRequestInterception(page);+ }+ };+ interceptedRequestsInProgress.on('delete', onDelete);+ interceptedRequestsInProgress.removeListener('delete', onDelete);+ }}};++async function disableRequestInterception(page) {+ await page.setRequestInterception(false);+ const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);+ page.removeListener('request', requestHandler);+}diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 888ad4438..23f94fb53 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -419,7 +419,9 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {const interceptRequestHandler = async (interceptedRequest) => {// We want to ensure that this won't get executed again in a case that there is a subsequent request// for example for some asset file link from main HTML.- if (wasCalled) return interceptedRequest.continue();+ if (wasCalled) {+ return interceptedRequest.continue();+ }wasCalled = true;const overrides = {};@@ -427,9 +429,8 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {if (method !== 'GET') overrides.method = method;if (payload) overrides.postData = payload;if (!_.isEmpty(headers)) overrides.headers = headers;-- await interceptedRequest.continue(overrides);- await removeInterceptRequestHandler(page, interceptRequestHandler); // We wan't this to be called only for the initial request.+ await removeInterceptRequestHandler(page, interceptRequestHandler);+ interceptedRequest.continue(overrides);};await addInterceptRequestHandler(page, interceptRequestHandler);diff --git a/test/puppeteer_request_interception.test.js b/test/puppeteer_request_interception.test.jsindex 1ee347f02..7c0f3b6b2 100644--- a/test/puppeteer_request_interception.test.js+++ b/test/puppeteer_request_interception.test.js@@ -133,6 +133,28 @@ describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptRequesawait browser.close();}});++ test('should allow async handler', async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });++ try {+ const page = await browser.newPage();++ await addInterceptRequestHandler(page, async (request) => {+ await Apify.utils.sleep(100);+ return request.continue({+ method: 'POST',+ });+ });++ // Check response that it's correct.+ const response = await page.goto('https://api.apify.com/v2/browser-info', { waitUntil: 'networkidle0' });+ const { method } = JSON.parse(await response.text());+ expect(method).toBe('POST');+ } finally {+ await browser.close();+ }+ });});describe('Apify.utils.puppeteer.removeInterceptRequestHandler()', () => {