Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): extract URLs with internationalized top-level domains (#4380)

SwastikTripathi019689a733ac
diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.ts
index 5099ead40..1c8989bae 100644
--- a/packages/utils/src/internals/general.ts
+++ b/packages/utils/src/internals/general.ts
@@ -5,14 +5,14 @@ import { setTimeout } from 'node:timers/promises';
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
*/
export const URL_NO_COMMAS_REGEX =
- /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
+ /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}\p{M}0-9@:%._+~#=]{0,254}[\p{L}\p{M}0-9])\.(xn--[a-z0-9-]{1,59}|[a-z]{2,63}|[\p{L}\p{M}]{2,63})(:\d{1,5})?(\/[-\p{L}\p{M}0-9@:%_+.~#?&/=()'*]*)?/giu;
/**
* Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
*/
export const URL_WITH_COMMAS_REGEX =
- /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
+ /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}\p{M}0-9@:%._+~#=]{0,254}[\p{L}\p{M}0-9])\.(xn--[a-z0-9-]{1,59}|[a-z]{2,63}|[\p{L}\p{M}]{2,63})(:\d{1,5})?(\/[-\p{L}\p{M}0-9@:%_+,.~#?&/=()'*]*)?/giu;
/**
* Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
diff --git a/test/utils/extract-urls.test.ts b/test/utils/extract-urls.test.ts
index 071cbdbcc..ad06c03c1 100644
--- a/test/utils/extract-urls.test.ts
+++ b/test/utils/extract-urls.test.ts
@@ -107,6 +107,25 @@ describe('extractUrls()', () => {
expect(extracted).toEqual([url]);
});
+ test('extracts URLs with internationalized top-level domains', () => {
+ const urls = [
+ 'https://пример.рф/путь',
+ 'https://xn--e1afmkfd.xn--p1ai/path',
+ 'https://例子.中国/',
+ 'https://उदाहरण.भारत/page',
+ 'https://test.みんな/',
+ ];
+ expect(extractUrls({ string: urls.join(' ') })).toEqual(urls);
+ expect(extractUrls({ string: urls.join(' '), urlRegExp: URL_WITH_COMMAS_REGEX })).toEqual(urls);
+ });
+
+ test('does not extend an ASCII top-level domain into adjacent non-Latin text', () => {
+ const string = '詳細はhttps://example.comをご覧ください 请访问https://www.google.com获取更多信息';
+ const expected = ['https://example.com', 'https://www.google.com'];
+ expect(extractUrls({ string })).toEqual(expected);
+ expect(extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX })).toEqual(expected);
+ });
+
test('does not extract invalid URLs', () => {
const { string } = getURLData(INVALID_URL_LIST);
const extracted = extractUrls({ string });