Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

add documentation comments

Ondrej Urban5ff39c67706d
diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.js
index 8285243e2..a5b551cf0 100644
--- a/src/puppeteer_live_view_client.js
+++ b/src/puppeteer_live_view_client.js
@@ -1,9 +1,28 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
+// everything gets destroyed slowly to make the UX better
const DESTROY_FADEOUT = 2000;
+/**
+ * Creates a page line in the Index.
+ *
+ * Used both client and server-side.
+ *
+ * @param {string} id
+ * @param {string} url
+ * @returns {string} html
+ */
const createPage = (id, url) => `<div class="page" id="${id}">URL: ${url}</div>`;
+/**
+ * Turns a Map of Pages into an Array of page divs.
+ * = All Pages of a single PuppeteerLiveViewBrowser.
+ *
+ * Used both client and server-side.
+ *
+ * @param {Map<String,Page>} pages
+ * @returns {string} html
+ */
const createPageCollection = (pages) => {
const pageDivs = [];
pages.forEach((page, id) => {
@@ -12,8 +31,26 @@ const createPageCollection = (pages) => {
return `<div class="page-collection">${pageDivs.join('\n')}</div>`;
};
+/**
+ * Creates a browser line in the Index.
+ *
+ * Used both client and server-side.
+ *
+ * @param {string} id Browser ID.
+ * @param {Map<String,Page>} pages
+ * @returns {string} html
+ */
const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>${id}</h3>${createPageCollection(pages)}</div>`;
+/**
+ * Turns a Set of Browsers into an Array of browser divs.
+ * = All PuppeteerLiveViewBrowsers of a PuppeteerLiveViewServer.
+ *
+ * Used both client and server-side.
+ *
+ * @param {Map<String,Page>} pages
+ * @returns {string} html
+ */
const createBrowserCollection = (browsers) => {
const browserDivs = [];
browsers.forEach(({ id, pages }) => {
@@ -23,18 +60,26 @@ const createBrowserCollection = (browsers) => {
};
/**
- * Generates a body for the Index page. A list of browsers and their pages.
+ * Generates an HTML body for the Index page.
+ * A list of PuppeteerLiveViewBrowsers and their Pages.
+ *
+ * Used only server-side.
+ *
* @param {Set} browsers
- * @returns {string} index page HTML
+ * @returns {string} html
*/
export const indexPage = (browsers) => {
return createBrowserCollection(browsers);
};
/**
- * Returns a body of a page consisting of a serialized image.
+ * Returns a body of a Page consisting of a serialized
+ * image and a HTML representation.
+ *
+ * Used only server-side.
+ *
* @param {Buffer} imageBuffer
- * @returns {string}
+ * @returns {string} html
*/
export const detailPage = ({ id, url, image, html }) => {
const chars = { '<': '&lt', '>': '&gt', '&': '&amp' };
@@ -56,12 +101,22 @@ export const detailPage = ({ id, url, image, html }) => {
`;
};
+/**
+ * The wsHandler() function encapsulates the whole client-side
+ * messaging and rendering logic. All commands that the client
+ * is able to receive and understand are listed in COMMANDS.
+ *
+ * @param {WebSocket} socket
+ */
const wsHandler = (socket) => {
+ // get common elements
const index = document.getElementById('index');
const pageDetail = document.getElementById('page-detail');
const backButton = document.getElementById('back-button');
+ // a client implementation of the server's sendCommand()
const sendCommand = (command, data) => {
+ // send only if socket is open
if (socket.readyState === 1) {
const payload = JSON.stringify({ command, data });
socket.send(payload);
@@ -69,6 +124,7 @@ const wsHandler = (socket) => {
};
const COMMANDS = {
+ // renders the Index Page - a list of browsers and their pages
renderIndex: ({ html }) => {
index.innerHTML = html;
index.querySelectorAll('.page').forEach((page) => {
@@ -77,22 +133,26 @@ const wsHandler = (socket) => {
});
});
},
+ // renders the Page Detail - where screenshots and HTML are shown
renderPage: ({ html }) => {
index.classList.add('hidden');
backButton.classList.remove('hidden');
pageDetail.classList.remove('hidden');
pageDetail.innerHTML = html;
},
+ // adds a browser to the list
createBrowser: ({ id }) => {
const browsers = index.querySelector('.browser-collection');
browsers.insertAdjacentHTML('afterbegin', createBrowser(id));
},
+ // removes a browser from the list after fade-out
destroyBrowser: ({ id }) => {
const browser = document.getElementById(id);
browser.classList.add('destroyed');
// do not remove immediately
setTimeout(() => browser.remove(), DESTROY_FADEOUT);
},
+ // adds a page to it's parent browser
createPage: ({ id, browserId, url }) => {
const pages = document.getElementById(browserId).querySelector('.page-collection');
pages.insertAdjacentHTML('afterbegin', createPage(id, url));
@@ -101,10 +161,12 @@ const wsHandler = (socket) => {
id: page.getAttribute('id'),
});
},
+ // updates page URL on navigation
updatePage: ({ id, url }) => {
const page = document.getElementById(id);
page.innerText = `URL: ${url}`;
},
+ // removes a page from a browser after fade-out
destroyPage: ({ id }) => {
const page = document.getElementById(id);
page.classList.add('destroyed');
@@ -112,6 +174,7 @@ const wsHandler = (socket) => {
// and the page only pops in the list for a split second
setTimeout(() => page.remove(), DESTROY_FADEOUT);
},
+ // handles errors
error: ({ message, status }) => {
console.error(`${status}: ${message}`); // eslint-disable-line
},
@@ -124,6 +187,8 @@ const wsHandler = (socket) => {
} catch (err) {
return console.error('Unable to parse message from server:', e.data); // eslint-disable-line
}
+
+ // validate and invoke requested command
const { command, data } = message;
if (!command) return console.error('Invalid command:', command); // eslint-disable-line
const fn = COMMANDS[command];
@@ -139,6 +204,11 @@ const wsHandler = (socket) => {
console.error(err); //eslint-disable-line
};
+ // The Index Page and Page Detail are not rerendered every time since
+ // it complicates WebSocket communication (adding / removing listeners).
+ // Instead, clicking on a page in the Index simply hides the Index
+ // and shows Page Detail and the Back Button.
+ // Clicking the back button hides the detail and shows the Index Page.
backButton.onclick = () => {
pageDetail.classList.add('hidden');
backButton.classList.add('hidden');
@@ -149,7 +219,11 @@ const wsHandler = (socket) => {
};
/**
- * Template for a basic layout of a HTML page.
+ * Template for a basic layout of a HTML page that enables WebSocket
+ * communication. Constants and JavaScript are templated directly into
+ * the HTML. This HTML page is the only piece of data sent over HTTP.
+ * All other communication takes place over WebSockets.
+ *
* @param {String} opts.host hostname of the WebSocket server
* @param {Number} opts.port port of the WebSocket server
* @returns {string} html
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
index 50a253c5a..2056da909 100644
--- a/src/puppeteer_live_view_server.js
+++ b/src/puppeteer_live_view_server.js
@@ -14,12 +14,14 @@ const DEFAULT_SCREENSHOT_TIMEOUT = 3000;
/**
* LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides
- * an API to safely get screenshots of the Browser's Pages.
+ * an API to capture screenshots and HTML of the Browser's Pages. It uses
+ * EventEmitter API to communicate relevant Page events.
+ *
* @param {Browser} browser A Puppeteer Browser instance.
* @param {String} [options.id] A unique ID of the LiveViewBrowser.
* @param {Number} [options.screenshotTimeout] Max time allowed for the screenshot taking process.
*/
-export class LiveViewBrowser extends EventEmitter {
+export class PuppeteerLiveViewBrowser extends EventEmitter {
constructor(browser, opts = {}) {
super();
this.browser = browser;
@@ -49,6 +51,7 @@ export class LiveViewBrowser extends EventEmitter {
page.on('load', () => {
this._loadedPages.add(page); // page is loaded
});
+ // using "framenavigated" on main Frame since "targetchanged" does not seem reliable
page.on('framenavigated', (frame) => {
if (frame === page.mainFrame()) {
this._loadedPages.delete(page); // page will load after nav
@@ -62,6 +65,7 @@ export class LiveViewBrowser extends EventEmitter {
.catch(err => log.error(err));
}
});
+ // clean up resources after page close
browser.on('targetdestroyed', (target) => {
if (target.type() === 'page') {
target.page()
@@ -75,11 +79,24 @@ export class LiveViewBrowser extends EventEmitter {
.catch(err => log.error(err));
}
});
+ // clean up resources after browser close (in Server)
browser.on('disconnected', b => this.emit('disconnected', b));
}
+ /**
+ * Initiates screenshot and HTML capturing of a Page by attaching
+ * a listener to Page "load" events. If the first Page is already loaded,
+ * it captures immediately.
+ *
+ * This function is invoked as a response to a Client side command "renderPage",
+ * which is issued by clicking on a Page listing on the Browser Index page.
+ * Once capturing starts, new screenshot and HTML will be served to Client
+ * with each Page load.
+ * @param {Page} page Puppeteer Page instance.
+ */
startCapturing(page) {
const id = this._pageIds.get(page);
+ if (!id) return;
const capture = () => {
log.debug(`Capturing page. ID: ${id}`);
this._getScreenshotAndHtml(page)
@@ -104,6 +121,15 @@ export class LiveViewBrowser extends EventEmitter {
});
}
+ /**
+ * Clears the capturing listener setup by startCapturing() to prevent
+ * unnecessary load on both Server and Client.
+ *
+ * Client side, the function is invoked by clicking the "back" button
+ * on a page detail (screenshot + HTML).
+ *
+ * @param {Page} page
+ */
stopCapturing(page) {
const id = this._pageIds.get(page);
this.removeAllListeners(id);
@@ -117,9 +143,11 @@ export class LiveViewBrowser extends EventEmitter {
* Therefore, the method prevents the page from being closed
* by replacing its close method and handling the page close
* itself once the screenshot has been taken.
+ *
* @param {Page} page Puppeteer's Page
* @returns {Promise<Buffer>} screenshot
* @private
+ * @ignore
*/
_getScreenshotAndHtml(page) {
// replace page's close function to prevent a close
@@ -191,12 +219,31 @@ export const startPuppeteerLiveView = (browserPromise, opts = {}) => {
return browserPromise
.then((browser) => {
- const lvb = new LiveViewBrowser(browser, browserOpts);
+ const lvb = new PuppeteerLiveViewBrowser(browser, browserOpts);
defaultServer.addBrowser(lvb);
return defaultServer;
});
};
+/**
+ * sendCommand() encapsulates simple WebSocket communication logic.
+ *
+ * @example
+ *
+ * A command is sent as JSON:
+ *
+ * {
+ * "command": "renderPage",
+ * "data": {
+ * "param1": "value1".
+ * "param2": "value2"
+ * }
+ * }
+ *
+ * @param {WebSocket} socket
+ * @param {String} command Name of requested command.
+ * @param {Object} data Data to be sent.
+ */
const sendCommand = (socket, command, data) => {
const payload = JSON.stringify({ command, data });
socket.send(payload, (err) => {
@@ -208,6 +255,7 @@ const sendCommand = (socket, command, data) => {
* Enables Live View monitoring of Act execution by spawning a web server that responds with a list
* of available browsers at its root path. Once the user chooses a browser, PuppeteerLiveViewServer will
* periodically serve screenshots of the selected browser's latest loaded page.
+ *
* @param {Number} [opts.port] Listening port of the PuppeteerLiveViewServer. Defaults to 1234.
*/
export default class PuppeteerLiveViewServer extends EventEmitter {
@@ -220,12 +268,23 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
this.httpServer = null;
}
+ /**
+ * Adds an instance of PuppeteerLiveViewBrowser (not Puppeteer.Browser)
+ * to the server's list of managed browsers.
+ *
+ * @param {PuppeteerLiveViewBrowser} browser
+ */
addBrowser(browser) {
this.browsers.add(browser);
this.emit('browsercreated', browser);
browser.on('disconnected', () => this.deleteBrowser(browser));
}
+ /**
+ * Removes an instance of PuppeteerLiveViewBrowser (not Puppeteer.Browser)
+ * from the server's list of managed browsers.
+ * @param {PuppeteerLiveViewBrowser} browser
+ */
deleteBrowser(browser) {
this.browsers.delete(browser);
this.emit('browserdestroyed', browser);
@@ -247,15 +306,19 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
}
/**
- * Request handler function that delegates to LiveViewRouter.
+ * Request handler function that returns a simple HTML index page
+ * with embedded JavaScript that will establish a WebSocket connection
+ * between the server and the client.
* @param {http.IncomingMessage} req
* @param {http.ServerResponse} res
* @private
+ * @ignore
*/
_httpRequestListener(req, res) {
const { port, address } = this.httpServer.address();
const body = layout({
+ // Node returns IPv6 when available by default, which doesn't work well with localhost.
host: address === LOCAL_IPV6 ? LOCAL_IPV4 : address,
port,
});
@@ -266,6 +329,13 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
res.end(body);
}
+ /**
+ * This function fires with each new WebSocket connection and manages
+ * the socket's messages.
+ * @param {WebSocket} ws
+ * @private
+ * @ignore
+ */
_wsRequestListener(ws) {
const BAD_REQUEST = {
message: 'Bad Request',
@@ -276,6 +346,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
status: 404,
};
+ // traverses browsers to find a page with ID
const findPage = (id) => {
let page;
let browser;
@@ -289,7 +360,6 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
return [browser, page];
};
-
log.debug('WebSocket connection to Puppeteer Live View established.');
ws.on('message', (msg) => {
try {
@@ -301,12 +371,14 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
if (command === 'renderIndex') {
sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });
+ // when the user selects a Page from the Index
} else if (command === 'renderPage') {
const { id } = msg.data || {};
const [browser, page] = findPage(id);
if (!page) return sendCommand(ws, 'error', NOT_FOUND);
browser.startCapturing(page);
browser.on(id, pageData => sendCommand(ws, 'renderPage', { html: detailPage(pageData) }));
+ // when the user clicks on the back button
} else if (command === 'quitPage') {
const { id } = msg.data || {};
const [browser, page] = findPage(id);
@@ -320,6 +392,13 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });
}
+ /**
+ * Invoked from the _wsRequestListener, this function only groups
+ * all browser related event handling into a single package.
+ * @param {WebSocket} ws
+ * @private
+ * @ignore
+ */
_setupCommandHandles(ws) {
const attachListeners = (browser) => {
const createListener = p => sendCommand(ws, 'createPage', p);
@@ -335,8 +414,10 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
browser.removeListener('pagenavigated', updateListener);
});
};
-
+ // Since this function fires when a socket opens, some browsers
+ // and pages already exist and we need to get messages about those too.
this.browsers.forEach(attachListeners);
+
const createListener = (browser) => {
attachListeners(browser);
sendCommand(ws, 'createBrowser', { id: browser.id });