Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): allow apostrophes and asterisks in extracted URLs (#3790)
Anas Khane9baffd78ec4
diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.tsindex e1cd8e7ef..6df1da439 100644--- a/packages/utils/src/internals/general.ts+++ b/packages/utils/src/internals/general.ts@@ -6,14 +6,14 @@ import { setTimeout } from 'node:timers/promises';* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).*/export const URL_NO_COMMAS_REGEX =- /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;+ /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;/*** Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.*/export const URL_WITH_COMMAS_REGEX =- /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;+ /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;let isDockerPromiseCache: Promise<boolean> | undefined;diff --git a/test/utils/extract-urls.test.ts b/test/utils/extract-urls.test.tsindex 99f80a7e0..d3c246e0d 100644--- a/test/utils/extract-urls.test.ts+++ b/test/utils/extract-urls.test.ts@@ -96,6 +96,19 @@ describe('extractUrls()', () => {expect(extracted).toEqual(array);});+ // https://github.com/apify/crawlee/issues/2755+ test('extracts URLs with apostrophes in the path', () => {+ const url = "https://www.zillow.com/homedetails/141-O'Canoe-Pl-Hertford-NC-27944/74398007_zpid/";+ const extracted = extractUrls({ string: url });+ expect(extracted).toEqual([url]);+ });++ test('extracts URLs with asterisks in the path', () => {+ const url = 'https://example.com/path*star/end';+ const extracted = extractUrls({ string: url });+ expect(extracted).toEqual([url]);+ });+test('does not extract invalid URLs', () => {const { string } = getURLData(INVALID_URL_LIST);const extracted = extractUrls({ string });