Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Remove xregexp dependency

Ondra Urbanadafe3f53e74
diff --git a/package.json b/package.json
index 957612f22..80a143288 100644
--- a/package.json
+++ b/package.json
@@ -68,8 +68,7 @@
"socket.io": "^2.2.0",
"tough-cookie": "^3.0.1",
"underscore": "^1.9.1",
- "ws": "^6.2.1",
- "xregexp": "^4.2.4"
+ "ws": "^6.2.1"
},
"devDependencies": {
"@apify/eslint-config": "^0.0.3",
diff --git a/src/utils.js b/src/utils.js
index d875dfa24..195c628e6 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -7,7 +7,6 @@ import fsExtra from 'fs-extra';
import ApifyClient from 'apify-client';
import psTree from '@apify/ps-tree';
import requestPromise from 'request-promise-native';
-import XRegExp from 'xregexp';
import cheerio from 'cheerio';
import log from 'apify-shared/log';
import semver from 'semver';
@@ -26,13 +25,13 @@ import { USER_AGENT_LIST } from './constants';
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
* @memberOf utils
*/
-const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
+const URL_NO_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
/**
* Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
* @memberOf utils
*/
-const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
+const URL_WITH_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
const ensureDirPromised = util.promisify(fsExtra.ensureDir);
const psTreePromised = util.promisify(psTree);