Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Include downloadListOfUrls() into RequestList; fix tests

Ondra Urbanec96632d8989
diff --git a/src/request_list.js b/src/request_list.js
index 929444d31..ede4342a1 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -6,14 +6,9 @@ import requestPromise from 'request-promise';
import Request from './request';
import events from './events';
import { ACTOR_EVENT_NAMES } from './constants';
-import { getFirstKey } from './utils';
+import { getFirstKey, downloadListOfUrls } from './utils';
import { getValue, setValue } from './key_value_store';
-// TODO: better tests
-// TODO: this will not accept diacritict chars, IMHO we should have here a regexp
-// that detects any hostname/whatever string
-const URL_REGEX = '(http|https)://[\\w-]+(\\.[\\w-]+)+([\\w-.,@?^=%&:/~+#-]*[\\w@?^=%&;/~+#-])?';
-
/**
* Helper function that validates unique.
* Throws an error if uniqueKey is not nonempty string.
@@ -356,14 +351,13 @@ export default class RequestList {
*/
_addRequestsFromUrl(source) {
const sharedOpts = _.omit(source, 'requestsFromUrl', 'regex');
- const {
- requestsFromUrl,
- regex = URL_REGEX,
- } = source;
-
- return requestPromise.get(requestsFromUrl)
- .then((urlsStr) => {
- const urlsArr = urlsStr.match(new RegExp(regex, 'gi'));
+ const { requestsFromUrl, regex } = source;
+
+ return downloadListOfUrls({
+ url: requestsFromUrl,
+ urlRegExp: regex,
+ })
+ .then((urlsArr) => {
const originalLength = this.requests.length;
if (urlsArr) {
diff --git a/src/utils.js b/src/utils.js
index 23a606502..49d733fc7 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -14,8 +14,8 @@ import { ENV_VARS } from './constants';
export const PID_USAGE_NOT_FOUND_ERROR = 'No maching pid found';
-export const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.~#?&//=]*)', 'gi');
-export const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.,~#?&//=]*)', 'gi');
+export const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)', 'gi'); // eslint-disable-line
+export const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.,~#?&//=\\(\\)]*)', 'gi'); // eslint-disable-line
const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);
const psTreePromised = Promise.promisify(psTree);
@@ -355,25 +355,14 @@ export const publicUtils = {
*
* @param {String} url
* @param {String} encoding
- * @param {Number} offset
- * @param {Number} limit
- * @param {Boolean} skipDuplicates
* @param {RegExp} urlRegExp
* @returns {Promise<Array>}
*/
-export const downloadListOfUrls = ({ url, encoding, offset, limit, skipDuplicates, urlRegExp }) => {
+export const downloadListOfUrls = ({ url, encoding, urlRegExp }) => {
if (!url) return Promise.reject(new Error('Resource URL must be provided.'));
encoding = encoding || 'utf8';
- offset = offset || 0;
return requestPromise.get({ url, encoding })
- .then((string) => {
- const urlsArr = extractUrls({ string, urlRegExp }); // not using keep commas for now
- const limited = urlsArr.slice(offset, limit || undefined);
- if (!skipDuplicates) return limited;
-
- const dupeSet = new Set();
- return limited.filter(u => (dupeSet.has(u) ? false : dupeSet.add(u)));
- });
+ .then(string => extractUrls({ string, urlRegExp }));
};
/**
@@ -383,7 +372,6 @@ export const downloadListOfUrls = ({ url, encoding, offset, limit, skipDuplicate
* @param keepCommas
* @returns {Array}
*/
-export const extractUrls = ({ string, urlRegExp, keepCommas }) => {
- const actualRegExp = urlRegExp || (keepCommas ? URL_WITH_COMMAS_REGEX : URL_NO_COMMAS_REGEX);
- return string.match(actualRegExp) || [];
+export const extractUrls = ({ string, urlRegExp = URL_NO_COMMAS_REGEX }) => {
+ return string.match(urlRegExp) || [];
};
diff --git a/test/data/lipsum.txt b/test/data/lipsum.txt
new file mode 100644
index 000000000..3b8bee76b
--- /dev/null
+++ b/test/data/lipsum.txt
@@ -0,0 +1,9 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce aliquet, nibh sit amet volutpat rhoncus, nisl eros sagittis sem, id semper quam augue eu mauris. Aliquam justo risus, condimentum ac urna quis, efficitur consectetur dui. Nam porta purus mi. Pellentesque non pellentesque eros. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Quisque ultricies in mi ac congue. Curabitur pretium non tortor non commodo. Cras scelerisque massa eros, eget ultricies urna tristique sed.
+
+Quisque vitae mattis felis, in dapibus erat. Vivamus eros elit, mattis et massa vel, laoreet convallis augue. Nunc vestibulum viverra sollicitudin. Maecenas nec eros a sem egestas consectetur id vitae lorem. In pellentesque lectus at nisi elementum rhoncus. Pellentesque venenatis justo et dictum consequat. Phasellus sollicitudin nisl tristique lacus maximus eleifend. Vestibulum vitae fringilla nulla. Integer et tincidunt lorem, semper malesuada arcu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Vestibulum blandit enim at pharetra lacinia. Nunc vitae turpis orci. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Maecenas nec libero in sapien tempus auctor vel in ligula.
+
+Integer et pulvinar lorem. Duis convallis volutpat ultrices. Nam id dignissim enim. Ut nec sem condimentum, aliquet arcu eget, dignissim tortor. Suspendisse mauris dolor, blandit sit amet molestie in, consectetur vitae sapien. Nulla euismod dapibus magna, in gravida justo consectetur sit amet. Nam semper orci ut sapien malesuada, et dictum nisl ultricies. Pellentesque sit amet ante tellus. Sed finibus tortor risus, in pulvinar enim iaculis a. Praesent varius vitae elit at dapibus. Suspendisse pretium, sapien nec laoreet pharetra, lectus eros vulputate nulla, a porta metus enim eu mi. Proin vehicula pulvinar nulla, commodo maximus velit scelerisque et. Quisque ornare, augue vel bibendum consequat, sapien metus convallis purus, vitae commodo quam elit finibus augue. Mauris sed orci id orci cursus volutpat. Fusce ut urna eu tellus fringilla consequat.
+
+Cras sit amet sem sit amet mi pulvinar varius non sed dolor. Nulla eu libero ultricies, aliquam mi eget, elementum velit. Aenean sed cursus orci. Cras aliquet dolor sit amet ipsum faucibus varius. Maecenas aliquam enim eu justo consequat cursus. Aenean nec velit a nulla facilisis tempus. Proin ac posuere leo. Maecenas dapibus, est in malesuada fermentum, augue dolor posuere purus, et scelerisque magna justo pulvinar eros. Phasellus faucibus imperdiet sem. Aliquam erat volutpat. Donec consequat eu metus non tristique. Ut vitae tincidunt risus. Nullam a quam laoreet, posuere leo id, rhoncus eros.
+
+Sed elit arcu, aliquam et tempor vel, ultrices sit amet turpis. Nullam et lorem magna. Nulla at risus nec dui aliquet ornare hendrerit id nulla. Donec sit amet porta ipsum. Sed blandit id metus et ornare. Vestibulum eleifend nunc id sagittis scelerisque. Lorem ipsum dolor sit amet, consectetur adipiscing elit. In tempor imperdiet erat consequat dapibus. Etiam gravida tellus risus, nec posuere nulla gravida ut. Praesent blandit interdum ipsum, sed pulvinar tellus varius convallis. Sed euismod ante id massa varius consequat. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Mauris et varius ex. Quisque placerat augue a libero malesuada volutpat. Maecenas aliquet mauris eget turpis dictum fermentum eget nec dolor.
diff --git a/test/data/tricky_url_list.txt b/test/data/tricky_url_list.txt
new file mode 100644
index 000000000..0c25fc24d
--- /dev/null
+++ b/test/data/tricky_url_list.txt
@@ -0,0 +1,21 @@
+http://foo.com/blah_blah
+http://foo.com/blah_blah/
+http://foo.com/blah_blah_(wikipedia)
+http://foo.com/blah_blah_(wikipedia)_(again)
+http://www.example.com/wpstyle/?p=364
+https://www.example.com/foo/?bar=baz&inga=42&quux
+http://userid:[email protected]:8080
+http://userid:[email protected]:8080/
+http://[email protected]:8080
+http://[email protected]:8080/
+http://userid:[email protected]
+http://userid:[email protected]/
+http://foo.com/blah_(wikipedia)#cite-1
+http://foo.com/blah_(wikipedia)_blah#cite-1
+http://foo.com/(something)?after=parens
+http://code.google.com/events/#&product=browser
+http://foo.bar/?q=Test%20URL-encoded%20stuff
+http://1337.net
+http://a.b-c.de
diff --git a/test/request_list.js b/test/request_list.js
index 9f2d44bbc..9805ac075 100644
--- a/test/request_list.js
+++ b/test/request_list.js
@@ -8,6 +8,7 @@ import { delayPromise } from 'apify-shared/utilities';
import Apify from '../build/index';
import * as keyValueStore from '../build/key_value_store';
import { ACTOR_EVENT_NAMES } from '../build/constants';
+import * as utils from '../build/utils';
chai.use(chaiAsPromised);
@@ -102,7 +103,7 @@ describe('Apify.RequestList', () => {
});
it('should correctly load list from hosted files in correct order', async () => {
- const mock = sinon.mock(request);
+ const mock = sinon.mock(utils);
const list1 = [
'https://example.com',
'https://google.com',
@@ -113,15 +114,15 @@ describe('Apify.RequestList', () => {
'https://page.com',
];
- mock.expects('get')
+ mock.expects('downloadListOfUrls')
.once()
- .withArgs('http://example.com/list-1')
- .returns(new Promise(resolve => setTimeout(resolve(list1.join('\n')), 100)));
+ .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })
+ .returns(new Promise(resolve => setTimeout(resolve(list1), 100)));
- mock.expects('get')
+ mock.expects('downloadListOfUrls')
.once()
- .withArgs('http://example.com/list-2')
- .returns(Promise.resolve(list2.join('\n')), 0);
+ .withArgs({ url: 'http://example.com/list-2', urlRegExp: undefined })
+ .returns(Promise.resolve(list2), 0);
const requestList = new Apify.RequestList({
sources: [
@@ -145,18 +146,20 @@ describe('Apify.RequestList', () => {
it('should use regex parameter to parse urls', async () => {
const mock = sinon.mock(request);
const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd';
- const listArr = ['https://example.com/a/b/c?q=1#abc', 'HTTP://google.com/a/b/c'];
+ const listArr = ['https://example.com', 'HTTP://google.com'];
mock.expects('get')
.once()
- .withArgs('http://example.com/list-1')
+ .withArgs({ url: 'http://example.com/list-1', encoding: 'utf8' })
.returns(Promise.resolve(listStr));
+ const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g;
const requestList = new Apify.RequestList({
sources: [
{
method: 'GET',
requestsFromUrl: 'http://example.com/list-1',
+ regex,
},
],
});
@@ -171,11 +174,11 @@ describe('Apify.RequestList', () => {
});
it('should handle requestsFromUrl with no URLs', async () => {
- const mock = sinon.mock(request);
- mock.expects('get')
+ const mock = sinon.mock(utils);
+ mock.expects('downloadListOfUrls')
.once()
- .withArgs('http://example.com/list-1')
- .returns(Promise.resolve('bla bla bla'));
+ .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })
+ .returns(Promise.resolve([]));
const requestList = new Apify.RequestList({
sources: [
diff --git a/test/utils.js b/test/utils.js
index 263f801af..55b68f246 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -6,9 +6,11 @@ import path from 'path';
import os from 'os';
import pidusage from 'pidusage';
import Promise from 'bluebird';
+import requestPromise from 'request-promise';
import * as utils from '../build/utils';
import Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
+import { URL_WITH_COMMAS_REGEX } from '../src/utils';
chai.use(chaiAsPromised);
@@ -346,59 +348,151 @@ describe('Apify.utils.sleep()', () => {
});
describe('utils.extractUrls()', () => {
- const simple = fs.readFileSync(path.join(__dirname, 'data', 'simple_url_list.txt'), 'utf8');
- const simpleArr = simple.trim().split(/[\r\n]+/g).map(u => u.trim());
- const unicode = fs.readFileSync(path.join(__dirname, 'data', 'unicode_url_list.txt'), 'utf8');
- const unicodeArr = unicode.trim().split(/[\r\n]+/g).map(u => u.trim());
- const withCommas = fs.readFileSync(path.join(__dirname, 'data', 'unicode+comma_url_list.txt'), 'utf8');
- const withCommasArr = withCommas.trim().split(/[\r\n]+/g).map(u => u.trim());
-
- const makeJSON = (string, array) => JSON.stringify({
+ const SIMPLE_URL_LIST = 'simple_url_list.txt';
+ const UNICODE_URL_LIST = 'unicode_url_list.txt';
+ const COMMA_URL_LIST = 'unicode+comma_url_list.txt';
+ const TRICKY_URL_LIST = 'tricky_url_list.txt';
+
+ const getURLData = (filename) => {
+ const string = fs.readFileSync(path.join(__dirname, 'data', filename), 'utf8');
+ const array = string.trim().split(/[\r\n]+/g).map(u => u.trim());
+ return { string, array };
+ };
+
+ const makeJSON = ({ string, array }) => JSON.stringify({
one: [{ http: string }],
two: array.map(url => ({ num: 123, url })),
});
const makeCSV = (array, delimiter) => array.map(url => ['ABC', 233, url, '.'].join(delimiter || ',')).join('\n');
+ const makeText = (array) => {
+ const text = fs.readFileSync(path.join(__dirname, 'data', 'lipsum.txt'), 'utf8').split('');
+ const ID = 'ů';
+ const maxIndex = text.length - 1;
+ array.forEach(() => {
+ const indexInText = Math.round(Math.random() * maxIndex);
+ if (text[indexInText] === ID) {
+ text[indexInText + 1] = ID;
+ } else {
+ text[indexInText] = ID;
+ }
+ });
+ return array.reduce((string, url) => string.replace(ID, ` ${url} `), text.join(''));
+ };
+
it('extracts simple URLs', () => {
- const extracted = utils.extractUrls({ string: simple });
- expect(extracted).to.be.eql(simpleArr);
+ const { string, array } = getURLData(SIMPLE_URL_LIST);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs', () => {
- const extracted = utils.extractUrls({ string: unicode });
- expect(extracted).to.be.eql(unicodeArr);
+ const { string, array } = getURLData(UNICODE_URL_LIST);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs with commas', () => {
- const extracted = utils.extractUrls({ string: withCommas, keepCommas: true });
- expect(extracted).to.be.eql(withCommasArr);
+ const { string, array } = getURLData(COMMA_URL_LIST);
+ const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ expect(extracted).to.be.eql(array);
+ });
+ it('extracts tricky URLs', () => {
+ const { string, array } = getURLData(TRICKY_URL_LIST);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
});
it('extracts simple URLs from JSON', () => {
- const string = makeJSON(simple, simpleArr);
+ const d = getURLData(SIMPLE_URL_LIST);
+ const string = makeJSON(d);
const extracted = utils.extractUrls({ string });
- expect(extracted).to.be.eql(simpleArr.concat(simpleArr));
+ expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts unicode URLs from JSON', () => {
- const string = makeJSON(unicode, unicodeArr);
+ const d = getURLData(UNICODE_URL_LIST);
+ const string = makeJSON(d);
const extracted = utils.extractUrls({ string });
- expect(extracted).to.be.eql(unicodeArr.concat(unicodeArr));
+ expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts unicode URLs with commas from JSON', () => {
- const string = makeJSON(withCommas, withCommasArr);
- const extracted = utils.extractUrls({ string, keepCommas: true });
- expect(extracted).to.be.eql(withCommasArr.concat(withCommasArr));
+ const d = getURLData(COMMA_URL_LIST);
+ const string = makeJSON(d);
+ const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ expect(extracted).to.be.eql(d.array.concat(d.array));
+ });
+ it('extracts tricky URLs from JSON', () => {
+ const d = getURLData(TRICKY_URL_LIST);
+ const string = makeJSON(d);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts simple URLs from CSV', () => {
- const string = makeCSV(simpleArr);
+ const { array } = getURLData(SIMPLE_URL_LIST);
+ const string = makeCSV(array);
const extracted = utils.extractUrls({ string });
- expect(extracted).to.be.eql(simpleArr);
+ expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs from CSV', () => {
- const string = makeCSV(unicodeArr);
+ const { array } = getURLData(UNICODE_URL_LIST);
+ const string = makeCSV(array);
const extracted = utils.extractUrls({ string });
- expect(extracted).to.be.eql(unicodeArr);
+ expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs with commas from semicolon CSV', () => {
- const string = makeCSV(withCommasArr, ';');
- const extracted = utils.extractUrls({ string, keepCommas: true });
- expect(extracted).to.be.eql(withCommasArr);
+ const { array } = getURLData(COMMA_URL_LIST);
+ const string = makeCSV(array, ';');
+ const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ expect(extracted).to.be.eql(array);
+ });
+ it('extracts tricky URLs from CSV', () => {
+ const { array } = getURLData(TRICKY_URL_LIST);
+ const string = makeCSV(array);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
+ });
+ it('extracts simple URLs from Text', () => {
+ const { array } = getURLData(SIMPLE_URL_LIST);
+ const string = makeText(array);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
+ });
+ it('extracts unicode URLs from Text', () => {
+ const { array } = getURLData(UNICODE_URL_LIST);
+ const string = makeText(array);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
+ });
+ it('extracts unicode URLs with commas from Text', () => {
+ const { array } = getURLData(COMMA_URL_LIST);
+ const string = makeText(array);
+ const extracted = utils.extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
+ expect(extracted).to.be.eql(array);
+ });
+ it('extracts tricky URLs from Text', () => {
+ const { array } = getURLData(TRICKY_URL_LIST);
+ const string = makeText(array);
+ const extracted = utils.extractUrls({ string });
+ expect(extracted).to.be.eql(array);
+ });
+});
+
+describe('utils.downloadListOfUrls()', () => {
+
+ let stub;
+
+ beforeEach(() => {
+ stub = sinon.stub(requestPromise, 'get');
+ });
+
+ afterEach(() => {
+ requestPromise.get.restore();
+ });
+
+ it('downloads a list of URLs', () => {
+ const text = fs.readFileSync(path.join(__dirname, 'data', 'simple_url_list.txt'), 'utf8');
+ const arr = text.trim().split(/[\r\n]+/g).map(u => u.trim());
+ stub.resolves(text);
+
+ return expect(utils.downloadListOfUrls({
+ url: 'nowhere',
+ })).to.eventually.deep.equal(arr);
});
});