Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: catch errors inside request interceptors (#1192)

Martin Adámek69a5a33ed57b
diff --git a/src/playwright_utils.js b/src/playwright_utils.js
index 87cc4ab83..a0b66d44a 100644
--- a/src/playwright_utils.js
+++ b/src/playwright_utils.js
@@ -56,20 +56,24 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {
log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance '
+ 'in recent versions of Playwright. Use only when necessary.');
let wasCalled = false;
- const interceptRequestHandler = (route) => {
- // We want to ensure that this won't get executed again in a case that there is a subsequent request
- // for example for some asset file link from main HTML.
- if (wasCalled) {
- return route.continue();
- }
+ const interceptRequestHandler = async (route) => {
+ try {
+ // We want to ensure that this won't get executed again in a case that there is a subsequent request
+ // for example for some asset file link from main HTML.
+ if (wasCalled) {
+ return route.continue();
+ }
- wasCalled = true;
- const overrides = {};
+ wasCalled = true;
+ const overrides = {};
- if (method !== 'GET') overrides.method = method;
- if (payload) overrides.postData = payload;
- if (!_.isEmpty(headers)) overrides.headers = headers;
- route.continue(overrides);
+ if (method !== 'GET') overrides.method = method;
+ if (payload) overrides.postData = payload;
+ if (!_.isEmpty(headers)) overrides.headers = headers;
+ await route.continue(overrides);
+ } catch (error) {
+ log.debug('Error inside request interceptor', { error });
+ }
};
await page.route('**/*', interceptRequestHandler);
diff --git a/src/puppeteer_request_interception.js b/src/puppeteer_request_interception.js
index f53deeb10..f592b1d53 100644
--- a/src/puppeteer_request_interception.js
+++ b/src/puppeteer_request_interception.js
@@ -2,6 +2,7 @@ import { EventEmitter } from 'events';
import ow from 'ow';
import _ from 'underscore';
import { Request as PuppeteerRequest, Page } from 'puppeteer'; // eslint-disable-line no-unused-vars
+import log from './utils_log';
// We use weak maps here so that the content gets discarted after page gets closed.
const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
@@ -232,10 +233,14 @@ export const removeInterceptRequestHandler = async (page, handler) => {
if (interceptedRequestsInProgress.size === 0) {
await disableRequestInterception(page);
} else {
- const onDelete = () => {
+ const onDelete = async () => {
if (interceptedRequestsInProgress.size === 0) {
- disableRequestInterception(page);
- interceptedRequestsInProgress.removeListener('delete', onDelete);
+ try {
+ await disableRequestInterception(page);
+ interceptedRequestsInProgress.removeListener('delete', onDelete);
+ } catch (error) {
+ log.debug('Error while disabling request interception', { error });
+ }
}
};
interceptedRequestsInProgress.on('delete', onDelete);