Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Redefine publicUtils exports; fix JSDoc comments to improve Docs

Ondra Urban63032d72af8f
diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.js
index 6a8526643..7c9b7ba25 100644
--- a/src/puppeteer_live_view_client.js
+++ b/src/puppeteer_live_view_client.js
@@ -11,6 +11,7 @@ const DESTROY_FADEOUT_MILLIS = 2000;
* @param {string} id
* @param {string} url
* @returns {string} html
+ * @ignore
*/
const createPage = (id, url) => `
<tr id="${id}">
@@ -28,6 +29,7 @@ const createPage = (id, url) => `
*
* @param {Map<String,Page>} pages
* @returns {string} html
+ * @ignore
*/
const createPageCollection = (pages) => {
const pageDivs = [];
@@ -54,6 +56,7 @@ const createPageCollection = (pages) => {
* @param {string} id Browser ID.
* @param {Map<String,Page>} pages
* @returns {string} html
+ * @ignore
*/
const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>Browser: ${id}</h3>${createPageCollection(pages)}</div>`;
@@ -65,6 +68,7 @@ const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>Browse
*
* @param {Set} browsers
* @returns {string} html
+ * @ignore
*/
const createBrowserCollection = (browsers) => {
const browserDivs = [];
@@ -82,6 +86,7 @@ const createBrowserCollection = (browsers) => {
*
* @param {Set} browsers
* @returns {string} html
+ * @ignore
*/
export const indexPage = (browsers) => {
return createBrowserCollection(browsers);
@@ -95,6 +100,7 @@ export const indexPage = (browsers) => {
*
* @param {Buffer} imageBuffer
* @returns {string} html
+ * @ignore
*/
export const detailPage = ({ id, url, image, html }) => {
const chars = { '<': '&lt', '>': '&gt', '&': '&amp' };
@@ -152,6 +158,7 @@ export const errorPage = ({ id, url, error }) => {
* is able to receive and understand are listed in COMMANDS.
*
* @param {WebSocket} socket
+ * @ignore
*/
const wsHandler = (socket) => {
// Get common elements
@@ -323,6 +330,7 @@ const wsHandler = (socket) => {
*
* @param {String} url Url of the WebSocket server
* @returns {string} html
+ * @ignore
*/
export const layout = (url) => {
checkParamOrThrow(url, 'url', 'String');
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
index 8335bbcbc..73804f42d 100644
--- a/src/puppeteer_live_view_server.js
+++ b/src/puppeteer_live_view_server.js
@@ -207,9 +207,9 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
}
}
-
+let liveViewServer;
/**
- * The start method should cover most use cases of starting a PuppeteerLiveViewServer.
+ * The registerBrowserForLiveView method should cover most use cases of starting a PuppeteerLiveViewServer.
* It creates a single instance of the server on its first invocation and subsequent
* invocations only add more browsers to the current server instance. Individual browsers
* are assigned unique IDs that will be used in displaying the browsers in an HTML index
@@ -226,7 +226,6 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
* @param {String} [opts.screenshotTimeoutMillis] Max time allowed for the screenshot taking process.
* @returns {Promise<PuppeteerLiveViewServer>}
*/
-let liveViewServer;
export const registerBrowserForLiveView = (browserPromise, opts = {}) => {
let serverPromise = Promise.resolve();
if (!liveViewServer) {
diff --git a/src/request_list.js b/src/request_list.js
index b95a9b023..af3560229 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -5,7 +5,7 @@ import Promise from 'bluebird';
import Request from './request';
import events from './events';
import { ACTOR_EVENT_NAMES } from './constants';
-import { getFirstKey, downloadListOfUrls } from './utils';
+import { getFirstKey, publicUtils } from './utils';
import { getValue, setValue } from './key_value_store';
/**
@@ -351,6 +351,7 @@ export default class RequestList {
_addRequestsFromUrl(source) {
const sharedOpts = _.omit(source, 'requestsFromUrl', 'regex');
const { requestsFromUrl, regex } = source;
+ const { downloadListOfUrls } = publicUtils;
return downloadListOfUrls({
url: requestsFromUrl,
diff --git a/src/utils.js b/src/utils.js
index 866901b2d..673289b8c 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -16,17 +16,17 @@ import { ENV_VARS } from './constants';
export const PID_USAGE_NOT_FOUND_ERROR = 'No maching pid found';
/**
- * Default regular expression to match URLs in a string, that may be plain text, JSON, CSV or other. It supports common URL characters
+ * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
* @memberOf utils
*/
-export const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
+const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
/**
* Regular expression that, in addition to the default regular expression URL_NO_COMMAS_REGEX, supports matching commas in URL path and query.
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
* @memberOf utils
*/
-export const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
+const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line
const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);
const psTreePromised = Promise.promisify(psTree);
@@ -348,10 +348,10 @@ const sleep = (millis) => {
* @param {String} url
* @param {String} [encoding='utf8']
* @param {RegExp} [urlRegExp=URL_NO_COMMAS_REGEX]
- * @returns {Promise<Array<string>>}
+ * @returns {Promise}
* @memberOf utils
*/
-export const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX }) => {
+const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX }) => {
try {
checkParamOrThrow(url, 'url', 'String');
checkParamOrThrow(encoding, 'string', 'String');
@@ -367,10 +367,10 @@ export const downloadListOfUrls = ({ url, encoding = 'utf8', urlRegExp = URL_NO_
* Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
* @param {String} string
* @param {RegExp} [urlRegExp=URL_NO_COMMAS_REGEX]
- * @returns {Array<string>}
+ * @returns {Array}
* @memberOf utils
*/
-export const extractUrls = ({ string, urlRegExp = URL_NO_COMMAS_REGEX }) => {
+const extractUrls = ({ string, urlRegExp = URL_NO_COMMAS_REGEX }) => {
checkParamOrThrow(string, 'string', 'String');
checkParamOrThrow(urlRegExp, 'urlRegExp', 'RegExp');
return string.match(urlRegExp) || [];
diff --git a/test/request_list.js b/test/request_list.js
index 9805ac075..45c104ba0 100644
--- a/test/request_list.js
+++ b/test/request_list.js
@@ -103,7 +103,7 @@ describe('Apify.RequestList', () => {
});
it('should correctly load list from hosted files in correct order', async () => {
- const mock = sinon.mock(utils);
+ const mock = sinon.mock(utils.publicUtils);
const list1 = [
'https://example.com',
'https://google.com',
@@ -174,7 +174,7 @@ describe('Apify.RequestList', () => {
});
it('should handle requestsFromUrl with no URLs', async () => {
- const mock = sinon.mock(utils);
+ const mock = sinon.mock(utils.publicUtils);
mock.expects('downloadListOfUrls')
.once()
.withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })
diff --git a/test/utils.js b/test/utils.js
index 78a87fa0f..9f32116d2 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -346,13 +346,15 @@ describe('Apify.utils.sleep()', () => {
});
});
-describe('utils.extractUrls()', () => {
+describe('Apify.utils.extractUrls()', () => {
const SIMPLE_URL_LIST = 'simple_url_list.txt';
const UNICODE_URL_LIST = 'unicode_url_list.txt';
const COMMA_URL_LIST = 'unicode+comma_url_list.txt';
const TRICKY_URL_LIST = 'tricky_url_list.txt';
const INVALID_URL_LIST = 'invalid_url_list.txt';
+ const { extractUrls, URL_WITH_COMMAS_REGEX } = utils.publicUtils;
+
const getURLData = (filename) => {
const string = fs.readFileSync(path.join(__dirname, 'data', filename), 'utf8');
const array = string.trim().split(/[\r\n]+/g).map(u => u.trim());
@@ -382,122 +384,123 @@ describe('utils.extractUrls()', () => {
it('extracts simple URLs', () => {
const { string, array } = getURLData(SIMPLE_URL_LIST);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs', () => {
const { string, array } = getURLData(UNICODE_URL_LIST);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs with commas', () => {
const { string, array } = getURLData(COMMA_URL_LIST);
- const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
+ const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(array);
});
it('extracts tricky URLs', () => {
const { string, array } = getURLData(TRICKY_URL_LIST);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('does not extract invalid URLs', () => {
const { string } = getURLData(INVALID_URL_LIST);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(['http://www.foo.bar']);
});
it('extracts simple URLs from JSON', () => {
const d = getURLData(SIMPLE_URL_LIST);
const string = makeJSON(d);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts unicode URLs from JSON', () => {
const d = getURLData(UNICODE_URL_LIST);
const string = makeJSON(d);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts unicode URLs with commas from JSON', () => {
const d = getURLData(COMMA_URL_LIST);
const string = makeJSON(d);
- const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
+ const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('extracts tricky URLs from JSON', () => {
const d = getURLData(TRICKY_URL_LIST);
const string = makeJSON(d);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(d.array.concat(d.array));
});
it('does not extract invalid URLs from JSON', () => {
const d = getURLData(INVALID_URL_LIST);
const string = makeJSON(d);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(['http://www.foo.bar', 'http://www.foo.bar']);
});
it('extracts simple URLs from CSV', () => {
const { array } = getURLData(SIMPLE_URL_LIST);
const string = makeCSV(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs from CSV', () => {
const { array } = getURLData(UNICODE_URL_LIST);
const string = makeCSV(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs with commas from semicolon CSV', () => {
const { array } = getURLData(COMMA_URL_LIST);
const string = makeCSV(array, ';');
- const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
+ const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(array);
});
it('extracts tricky URLs from CSV', () => {
const { array } = getURLData(TRICKY_URL_LIST);
const string = makeCSV(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('does not extract invalid URLs from CSV', () => {
const { array } = getURLData(INVALID_URL_LIST);
const string = makeCSV(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(['http://www.foo.bar']);
});
it('extracts simple URLs from Text', () => {
const { array } = getURLData(SIMPLE_URL_LIST);
const string = makeText(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs from Text', () => {
const { array } = getURLData(UNICODE_URL_LIST);
const string = makeText(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('extracts unicode URLs with commas from Text', () => {
const { array } = getURLData(COMMA_URL_LIST);
const string = makeText(array);
- const extracted = utils.extractUrls({ string, urlRegExp: utils.URL_WITH_COMMAS_REGEX });
+ const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).to.be.eql(array);
});
it('extracts tricky URLs from Text', () => {
const { array } = getURLData(TRICKY_URL_LIST);
const string = makeText(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(array);
});
it('does not extract invalid URLs from Text', () => {
const { array } = getURLData(INVALID_URL_LIST);
const string = makeText(array);
- const extracted = utils.extractUrls({ string });
+ const extracted = extractUrls({ string });
expect(extracted).to.be.eql(['http://www.foo.bar']);
});
});
-describe('utils.downloadListOfUrls()', () => {
+describe('Apify.utils.downloadListOfUrls()', () => {
+ const { downloadListOfUrls } = utils.publicUtils;
let stub;
beforeEach(() => {
stub = sinon.stub(requestPromise, 'get');
@@ -511,7 +514,7 @@ describe('utils.downloadListOfUrls()', () => {
const arr = text.trim().split(/[\r\n]+/g).map(u => u.trim());
stub.resolves(text);
- return expect(utils.downloadListOfUrls({
+ return expect(downloadListOfUrls({
url: 'nowhere',
})).to.eventually.deep.equal(arr);
});