Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add tests for invalid URLs. Update RegExp

Ondra Urban242160bef512
diff --git a/src/utils.js b/src/utils.js
index a0506f8a9..705732642 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -20,13 +20,13 @@ export const PID_USAGE_NOT_FOUND_ERROR = 'No maching pid found';
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
* @memberOf utils
*/
-export const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)', 'gi'); // eslint-disable-line
+export const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
/**
* Regular expression that, in addition to the default regular expression URL_NO_COMMAS_REGEX, supports matching commas in URL path and query.
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
* @memberOf utils
*/
-export const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.,~#?&//=\\(\\)]*)', 'gi'); // eslint-disable-line
+export const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);
const psTreePromised = Promise.promisify(psTree);
@@ -355,7 +355,7 @@ export const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_
try {
checkParamOrThrow(string, 'string', 'String');
checkParamOrThrow(encoding, 'string', 'String');
- checkParamOrThrow(urlRegExp, 'urlRegExp', 'Object');
+ checkParamOrThrow(urlRegExp, 'urlRegExp', 'RegExp');
} catch (err) {
return Promise.reject(err);
}
@@ -372,7 +372,7 @@ export const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_
*/
export const extractUrls = ({ string, urlRegExp = URL_NO_COMMAS_REGEX }) => {
checkParamOrThrow(string, 'string', 'String');
- checkParamOrThrow(urlRegExp, 'urlRegExp', 'Object');
+ checkParamOrThrow(urlRegExp, 'urlRegExp', 'RegExp');
return string.match(urlRegExp) || [];
};
diff --git a/test/data/invalid_url_list.txt b/test/data/invalid_url_list.txt
new file mode 100644
index 000000000..fb3353f83
--- /dev/null
+++ b/test/data/invalid_url_list.txt
@@ -0,0 +1,29 @@
+http://
+http://.
+http://..
+http://../
+http://?
+http://??
+http://??/
+http://#
+http://##
+http://##/
+//
+//a
+///a
+///
+http:///a
+foo.com
+rdar://1234
+h://test
+http:// shouldfail.com
+:// should fail
+ftps://foo.bar/
+http://-error-.invalid/
+http://-a.b.co
+http://a.b-.co
+http://0.0.0.0
+http://3628126748
+http://.www.foo.bar/
+http://www.foo.bar./
+http://.www.foo.bar./)
diff --git a/test/utils.js b/test/utils.js
index 55b68f246..087f5503d 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -10,7 +10,6 @@ import requestPromise from 'request-promise';
import * as utils from '../build/utils';
import Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
-import { URL_WITH_COMMAS_REGEX } from '../src/utils';
chai.use(chaiAsPromised);
@@ -352,6 +351,7 @@ describe('utils.extractUrls()', () => {
const UNICODE_URL_LIST = 'unicode_url_list.txt';
const COMMA_URL_LIST = 'unicode+comma_url_list.txt';
const TRICKY_URL_LIST = 'tricky_url_list.txt';
+ const INVALID_URL_LIST = 'invalid_url_list.txt';
const getURLData = (filename) => {
const string = fs.readFileSync(path.join(__dirname, 'data', filename), 'utf8');
@@ -392,7 +392,7 @@ describe('utils.extractUrls()', () => {
});
it('extracts unicode URLs with commas', () => {
const { string, array } = getURLData(COMMA_URL_LIST);
- const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(array);
});
it('extracts tricky URLs', () => {
@@ -400,6 +400,11 @@ describe('utils.extractUrls()', () => {
const extracted = utils.extractUrls({ string });
expect(extracted).to.be.eql(array);
});
+ it('does not extract invalid URLs', () => {
+ const { string } = getURLData(INVALID_URL_LIST);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(['http://www.foo.bar']);
+ });
it('extracts simple URLs from JSON', () => {
const d = getURLData(SIMPLE_URL_LIST);
const string = makeJSON(d);
@@ -415,7 +420,7 @@ describe('utils.extractUrls()', () => {
it('extracts unicode URLs with commas from JSON', () => {
const d = getURLData(COMMA_URL_LIST);
const string = makeJSON(d);
- const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts tricky URLs from JSON', () => {
@@ -424,6 +429,12 @@ describe('utils.extractUrls()', () => {
const extracted = utils.extractUrls({ string });
expect(extracted).to.be.eql(d.array.concat(d.array));
});
+ it('does not extract invalid URLs from JSON', () => {
+ const d = getURLData(INVALID_URL_LIST);
+ const string = makeJSON(d);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(['http://www.foo.bar', 'http://www.foo.bar']);
+ });
it('extracts simple URLs from CSV', () => {
const { array } = getURLData(SIMPLE_URL_LIST);
const string = makeCSV(array);
@@ -439,7 +450,7 @@ describe('utils.extractUrls()', () => {
it('extracts unicode URLs with commas from semicolon CSV', () => {
const { array } = getURLData(COMMA_URL_LIST);
const string = makeCSV(array, ';');
- const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(array);
});
it('extracts tricky URLs from CSV', () => {
@@ -448,6 +459,12 @@ describe('utils.extractUrls()', () => {
const extracted = utils.extractUrls({ string });
expect(extracted).to.be.eql(array);
});
+ it('does not extract invalid URLs from CSV', () => {
+ const { array } = getURLData(INVALID_URL_LIST);
+ const string = makeCSV(array);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(['http://www.foo.bar']);
+ });
it('extracts simple URLs from Text', () => {
const { array } = getURLData(SIMPLE_URL_LIST);
const string = makeText(array);
@@ -463,7 +480,7 @@ describe('utils.extractUrls()', () => {
it('extracts unicode URLs with commas from Text', () => {
const { array } = getURLData(COMMA_URL_LIST);
const string = makeText(array);
- const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(array);
});
it('extracts tricky URLs from Text', () => {
@@ -472,6 +489,12 @@ describe('utils.extractUrls()', () => {
const extracted = utils.extractUrls({ string });
expect(extracted).to.be.eql(array);
});
+ it('does not extract invalid URLs from Text', () => {
+ const { array } = getURLData(INVALID_URL_LIST);
+ const string = makeText(array);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(['http://www.foo.bar']);
+ });
});
describe('utils.downloadListOfUrls()', () => {