Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
"Add downloadListOfUrls and extractUrls functions to utils"
Ondra Urbancd797ad3cfe5
diff --git a/package.json b/package.jsonindex 79084d335..a89e52452 100644--- a/package.json+++ b/package.json@@ -55,7 +55,8 @@"ps-tree": "^1.1.0","request-promise": "^4.2.2","underscore": "^1.9.0",- "ws": "^5.1.0"+ "ws": "^5.1.0",+ "xregexp": "^4.2.0"},"devDependencies": {"apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",diff --git a/src/utils.js b/src/utils.jsindex 82b0bc68a..23a606502 100644--- a/src/utils.js+++ b/src/utils.js@@ -6,12 +6,17 @@ import fsExtra from 'fs-extra';import ApifyClient from 'apify-client';import psTree from 'ps-tree';import pidusage from 'pidusage';+import requestPromise from 'request-promise';import _ from 'underscore';+import XRegExp from 'xregexp';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS } from './constants';export const PID_USAGE_NOT_FOUND_ERROR = 'No maching pid found';+export const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.~#?&//=]*)', 'gi');+export const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[-\\p{L}0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-\\p{L}0-9@:%_\\+.,~#?&//=]*)', 'gi');+const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);const psTreePromised = Promise.promisify(psTree);@@ -343,3 +348,42 @@ const sleep = (millis) => {export const publicUtils = {sleep,};++/**+ * Returns a promise that resolves to an array of urls parsed from the resource+ * available at the provided url.+ *+ * @param {String} url+ * @param {String} encoding+ * @param {Number} offset+ * @param {Number} limit+ * @param {Boolean} skipDuplicates+ * @param {RegExp} urlRegExp+ * @returns {Promise<Array>}+ */+export const downloadListOfUrls = ({ url, encoding, offset, limit, skipDuplicates, urlRegExp }) => {+ if (!url) return Promise.reject(new Error('Resource URL must be provided.'));+ encoding = encoding || 'utf8';+ offset = offset || 0;+ return requestPromise.get({ url, encoding })+ .then((string) => {+ const urlsArr = extractUrls({ string, urlRegExp }); // not using keep commas for now+ const limited = urlsArr.slice(offset, limit || undefined);+ if (!skipDuplicates) return limited;++ const dupeSet = new Set();+ return limited.filter(u => (dupeSet.has(u) ? false : dupeSet.add(u)));+ });+};++/**+ * Collects all URLs in an arbitrary string to an array.+ * @param string+ * @param urlRegExp+ * @param keepCommas+ * @returns {Array}+ */+export const extractUrls = ({ string, urlRegExp, keepCommas }) => {+ const actualRegExp = urlRegExp || (keepCommas ? URL_WITH_COMMAS_REGEX : URL_NO_COMMAS_REGEX);+ return string.match(actualRegExp) || [];+};diff --git a/test/data/simple_url_list.txt b/test/data/simple_url_list.txtnew file mode 100644index 000000000..f15af531b--- /dev/null+++ b/test/data/simple_url_list.txt@@ -0,0 +1,100 @@+ http://example.com/boundary/babies.html+ http://example.com/birthday/bite.php+ https://www.example.com/#birth+ https://www.example.net/bag?bedroom=box+ http://basketball.example.com/books#activity+ http://example.edu/blood+ http://www.example.edu/afterthought?brick=believe&afterthought=addition+ http://arm.example.com/+ https://www.example.com/berry+ http://example.com/border/alarm.php+ http://example.net/+ https://www.example.com/+ http://www.example.com/brick/addition+ https://example.com/bed.html+ http://www.example.org/airplane+ https://www.example.com/#babies+ https://www.example.com/amount+ http://www.example.com/+ http://bubble.example.com/advertisement.php+ http://example.net/+ http://adjustment.example.com/bikes/boy.html+ http://example.com/+ https://www.example.com/beginner/battle+ http://bee.example.com/art/birth+ https://www.example.com/bubble/amount.php?ants=brass&approval=brother#bite+ http://www.example.net/?beds=bone+ http://www.example.com/+ http://www.example.com/+ http://anger.example.org/bone/action.htm?baby=balance+ https://www.example.com/apparel+ https://www.example.com/badge/bike+ http://www.example.com/apparel+ https://border.example.edu/+ https://www.example.com/+ https://arm.example.com/baseball/bird.htm?animal=amusement+ http://www.example.com/animal.html+ http://example.com/arm.php+ https://www.example.org/+ http://bubble.example.com/basketball.php?ants=bed&believe=beef#box+ https://www.example.org/#anger+ http://apparatus.example.edu/argument+ https://www.example.com/+ http://www.example.com/+ http://www.example.com/art.aspx+ https://www.example.com/agreement/brother.aspx?attraction=bite+ http://anger.example.com/?account=bottle&bath=behavior+ https://www.example.com/#acoustics+ https://www.example.com/addition.php+ https://www.example.com/brass+ https://www.example.com/?branch=blow&actor=bath+ http://www.example.com/blow.php+ http://example.com/authority+ http://www.example.com/birthday.html#bed+ http://boot.example.com/bottle?agreement=approval&believe=army+ https://www.example.com/+ http://www.example.com/+ https://example.edu/action/achiever+ https://www.example.com/branch/book.php?brass=babies+ http://www.example.com/+ https://back.example.edu/activity+ https://www.example.edu/birth.php+ http://adjustment.example.com/book+ http://www.example.com/?book=baby&bait=board+ http://www.example.com/+ https://example.com/+ https://www.example.com/+ http://www.example.com/#behavior+ https://www.example.net/base+ https://example.com/act+ http://www.example.edu/+ http://www.example.net/+ https://www.example.com/?art=bear+ https://www.example.com/basin/bed.aspx#act+ http://www.example.com/+ http://example.net/baseball/addition?blood=blade&apparatus=anger+ https://www.example.com/afternoon/appliance.html+ https://apparatus.example.com/bedroom.html+ https://www.example.com/+ https://www.example.com/+ http://www.example.com/birds+ http://example.org/?boat=believe+ https://www.example.com/#angle+ https://www.example.com/+ https://example.com/#believe+ http://birthday.example.com/bottle?activity=alarm&bite=bottle+ https://example.net/+ https://achiever.example.com/+ http://www.example.com/boundary/believe+ http://example.com/appliance/base.html+ http://example.com/bikes.php+ http://base.example.com/+ http://www.example.com/+ http://www.example.com/+ https://www.example.com/base?birth=basketball&afternoon=actor+ http://www.example.net/bead+ https://www.example.org/?alarm=birds+ https://www.example.com/art+ https://www.example.com/+ https://example.net/action/box.php+ http://www.example.com/diff --git a/test/data/unicode+comma_url_list.txt b/test/data/unicode+comma_url_list.txtnew file mode 100644index 000000000..e550b10b1--- /dev/null+++ b/test/data/unicode+comma_url_list.txt@@ -0,0 +1,11 @@+http://www.astro.com/astro-databank/Abrahams,_Jim_S.+http://www.astro.com/astro-databank/Arnaz,_Desi_Jr.+http://www.astro.com/astro-databank/Dempster,_Kenneth_C.+https://www.astro.com/astro-databank/Fillon,_François+http://www.astro.com/astro-databank/Fucher,_Waldir_B.+http://www.astro.com/astro-databank/Kennedy,_David_M.+http://www.astro.com/astro-databank/McCarthy,_James_F.+http://www.astro.com/astro-databank/Eshelman,_James_A.+http://www.astro.com/astro-databank/Cohana,_Mélissande+http://www.astro.com/astro-databank/Jauch,_Günther+http://www.astro.com/astro-databank/Judelson,_David_N.diff --git a/test/data/unicode_url_list.txt b/test/data/unicode_url_list.txtnew file mode 100644index 000000000..5df978c85--- /dev/null+++ b/test/data/unicode_url_list.txt@@ -0,0 +1,4 @@+http://www.example.com/düsseldorf?neighbourhood=Lörick+http://ko.wikipedia.org/wiki/위키백과:대문+http://en.wikipedia.org/wiki/ɸ+http://stackoverflow.com/questions/2742852/これはこれを日本語のテキストですdiff --git a/test/utils.js b/test/utils.jsindex 3a2659a5f..263f801af 100644--- a/test/utils.js+++ b/test/utils.js@@ -2,6 +2,7 @@ import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import sinon from 'sinon';import fs from 'fs';+import path from 'path';import os from 'os';import pidusage from 'pidusage';import Promise from 'bluebird';@@ -343,3 +344,61 @@ describe('Apify.utils.sleep()', () => {});});});++describe('utils.extractUrls()', () => {+ const simple = fs.readFileSync(path.join(__dirname, 'data', 'simple_url_list.txt'), 'utf8');+ const simpleArr = simple.trim().split(/[\r\n]+/g).map(u => u.trim());+ const unicode = fs.readFileSync(path.join(__dirname, 'data', 'unicode_url_list.txt'), 'utf8');+ const unicodeArr = unicode.trim().split(/[\r\n]+/g).map(u => u.trim());+ const withCommas = fs.readFileSync(path.join(__dirname, 'data', 'unicode+comma_url_list.txt'), 'utf8');+ const withCommasArr = withCommas.trim().split(/[\r\n]+/g).map(u => u.trim());++ const makeJSON = (string, array) => JSON.stringify({+ one: [{ http: string }],+ two: array.map(url => ({ num: 123, url })),+ });+ const makeCSV = (array, delimiter) => array.map(url => ['ABC', 233, url, '.'].join(delimiter || ',')).join('\n');++ it('extracts simple URLs', () => {+ const extracted = utils.extractUrls({ string: simple });+ expect(extracted).to.be.eql(simpleArr);+ });+ it('extracts unicode URLs', () => {+ const extracted = utils.extractUrls({ string: unicode });+ expect(extracted).to.be.eql(unicodeArr);+ });+ it('extracts unicode URLs with commas', () => {+ const extracted = utils.extractUrls({ string: withCommas, keepCommas: true });+ expect(extracted).to.be.eql(withCommasArr);+ });+ it('extracts simple URLs from JSON', () => {+ const string = makeJSON(simple, simpleArr);+ const extracted = utils.extractUrls({ string });+ expect(extracted).to.be.eql(simpleArr.concat(simpleArr));+ });+ it('extracts unicode URLs from JSON', () => {+ const string = makeJSON(unicode, unicodeArr);+ const extracted = utils.extractUrls({ string });+ expect(extracted).to.be.eql(unicodeArr.concat(unicodeArr));+ });+ it('extracts unicode URLs with commas from JSON', () => {+ const string = makeJSON(withCommas, withCommasArr);+ const extracted = utils.extractUrls({ string, keepCommas: true });+ expect(extracted).to.be.eql(withCommasArr.concat(withCommasArr));+ });+ it('extracts simple URLs from CSV', () => {+ const string = makeCSV(simpleArr);+ const extracted = utils.extractUrls({ string });+ expect(extracted).to.be.eql(simpleArr);+ });+ it('extracts unicode URLs from CSV', () => {+ const string = makeCSV(unicodeArr);+ const extracted = utils.extractUrls({ string });+ expect(extracted).to.be.eql(unicodeArr);+ });+ it('extracts unicode URLs with commas from semicolon CSV', () => {+ const string = makeCSV(withCommasArr, ';');+ const extracted = utils.extractUrls({ string, keepCommas: true });+ expect(extracted).to.be.eql(withCommasArr);+ });+});