Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Refactor PuppeteerLiveView to improve readability

Ondra Urban81cde489f5ba
diff --git a/package.json b/package.json
index 651eda0d0..3ef9ddd2b 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,6 @@
"ps-tree": "^1.1.0",
"request-promise": "^4.2.2",
"underscore": "^1.9.0",
- "uuid": "^3.2.1",
"ws": "^5.1.0"
},
"devDependencies": {
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 2ce49501f..e3b52e909 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -39,6 +39,12 @@ const LAUNCH_PUPPETEER_LOG_OMIT_OPTS = [
* will use the same target proxy server (i.e. the same IP address).
* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* Only applied if the `useApifyProxy` option is `true`.
+ * @property {Boolean} [opts.liveView=false] If set to `true`, a PuppeteerLiveViewServer will be started to enable
+ * screenshot and html capturing of visited pages using PuppeteerLiveViewBrowser.
+ * @property {Object} [opts.liveViewOptions] Settings for PuppeteerLiveViewBrowser started using launchPuppeteer().
+ * @property {String} [opts.liveViewOptions.id] Custom ID of a browser instance.
+ * @property {Number} [opts.liveViewOptions.screenshotTimeoutMillis] Time in milliseconds before a screenshot capturing
+ * will time out and the actor continues with execution. Screenshot capturing pauses execution within the given page.
*/
/**
@@ -157,6 +163,8 @@ export const launchPuppeteer = (opts = {}) => {
checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');
checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'Maybe String');
checkParamOrThrow(opts.useApifyProxy, 'opts.useApifyProxy', 'Maybe Boolean');
+ checkParamOrThrow(opts.liveView, 'opts.liveView', 'Maybe Boolean');
+ checkParamOrThrow(opts.liveViewOptions, 'opts.liveViewOptoins', 'Maybe Object');
if (opts.useApifyProxy && opts.proxyUrl) throw new Error('Cannot combine "opts.useApifyProxy" with "opts.proxyUrl"!');
const puppeteer = getPuppeteerOrThrow();
@@ -165,7 +173,7 @@ export const launchPuppeteer = (opts = {}) => {
optsCopy.args = optsCopy.args || [];
optsCopy.args.push('--no-sandbox');
if (optsCopy.headless == null) {
- // forcing headless with liveView, otherwise screenshots redirect user to new browser window
+ // Forcing headless with liveView, otherwise screenshots redirect user to new browser window
optsCopy.headless = optsCopy.liveView || (process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1');
}
if (optsCopy.useChrome && (optsCopy.executablePath === undefined || optsCopy.executablePath === null)) {
@@ -201,8 +209,8 @@ export const launchPuppeteer = (opts = {}) => {
// Ensure that the returned promise is of type Bluebird.
const wrapped = newPromise().then(() => browserPromise);
- // start LiveView server if requested
- if (optsCopy.liveView) return registerBrowserForLiveView(wrapped, optsCopy.liveViewOptions).then(() => wrapped); // TODO document Options
+ // Start LiveView server if requested
+ if (optsCopy.liveView) return registerBrowserForLiveView(wrapped, optsCopy.liveViewOptions).then(() => wrapped);
return wrapped;
};
diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.js
index 695dda470..c27029219 100644
--- a/src/puppeteer_live_view_client.js
+++ b/src/puppeteer_live_view_client.js
@@ -1,7 +1,7 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
-// everything gets destroyed slowly to make the UX better
-const DESTROY_FADEOUT = 2000;
+// Everything gets destroyed slowly to make the UX better
+const DESTROY_FADEOUT_MILLIS = 2000;
/**
* Creates a page line in the Index.
@@ -48,7 +48,7 @@ const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>${id}<
*
* Used both client and server-side.
*
- * @param {Map<String,Page>} pages
+ * @param {Set} browsers
* @returns {string} html
*/
const createBrowserCollection = (browsers) => {
@@ -121,14 +121,14 @@ export const errorPage = ({ id, url, error }) => {
* @param {WebSocket} socket
*/
const wsHandler = (socket) => {
- // get common elements
+ // Get common elements
const index = document.getElementById('index');
const pageDetail = document.getElementById('page-detail');
const backButton = document.getElementById('back-button');
- // a client implementation of the server's sendCommand()
+ // A client implementation of the server's sendCommand()
const sendCommand = (command, data) => {
- // send only if socket is open
+ // Send only if socket is open
if (socket.readyState === 1) {
const payload = JSON.stringify({ command, data });
socket.send(payload);
@@ -136,7 +136,7 @@ const wsHandler = (socket) => {
};
const COMMANDS = {
- // renders the Index Page - a list of browsers and their pages
+ // Renders the Index Page - a list of browsers and their pages
renderIndex: ({ html }) => {
index.innerHTML = html;
index.querySelectorAll('.page').forEach((page) => {
@@ -145,26 +145,26 @@ const wsHandler = (socket) => {
});
});
},
- // renders the Page Detail - where screenshots and HTML are shown
+ // Renders the Page Detail - where screenshots and HTML are shown
renderPage: ({ html }) => {
index.classList.add('hidden');
backButton.classList.remove('hidden');
pageDetail.classList.remove('hidden');
pageDetail.innerHTML = html;
},
- // adds a browser to the list
+ // Adds a browser to the list
createBrowser: ({ id }) => {
const browsers = index.querySelector('.browser-collection');
browsers.insertAdjacentHTML('afterbegin', createBrowser(id));
},
- // removes a browser from the list after fade-out
+ // Removes a browser from the list after fade-out
destroyBrowser: ({ id }) => {
const browser = document.getElementById(id);
browser.classList.add('destroyed');
- // do not remove immediately
- setTimeout(() => browser.remove(), DESTROY_FADEOUT);
+ // Do not remove immediately
+ setTimeout(() => browser.remove(), DESTROY_FADEOUT_MILLIS);
},
- // adds a page to it's parent browser
+ // Adds a page to it's parent browser
createPage: ({ id, browserId, url }) => {
const pages = document.getElementById(browserId).querySelector('.page-collection');
pages.insertAdjacentHTML('afterbegin', createPage(id, url));
@@ -173,20 +173,20 @@ const wsHandler = (socket) => {
id: page.getAttribute('id'),
});
},
- // updates page URL on navigation
+ // Updates page URL on navigation
updatePage: ({ id, url }) => {
const page = document.getElementById(id);
page.innerText = `URL: ${url}`;
},
- // removes a page from a browser after fade-out
+ // Removes a page from a browser after fade-out
destroyPage: ({ id }) => {
const page = document.getElementById(id);
page.classList.add('destroyed');
- // do not remove immediately since it can happen pretty fast
- // and the page only pops in the list for a split second
- setTimeout(() => page.remove(), DESTROY_FADEOUT);
+ // Do not remove immediately since it can happen pretty fast
+ // and the page only pops in the list for a split second.
+ setTimeout(() => page.remove(), DESTROY_FADEOUT_MILLIS);
},
- // handles errors
+ // Handles errors
error: ({ message, status }) => {
console.error(`${status}: ${message}`); // eslint-disable-line
},
@@ -200,7 +200,7 @@ const wsHandler = (socket) => {
return console.error('Unable to parse message from server:', e.data); // eslint-disable-line
}
- // validate and invoke requested command
+ // Validate and invoke requested command
const { command, data } = message;
if (!command) return console.error('Invalid command:', command); // eslint-disable-line
const fn = COMMANDS[command];
@@ -258,7 +258,7 @@ export const layout = (url) => {
color: #660000;
visibility: hidden;
opacity: 0;
- transition: visibility 0s ${DESTROY_FADEOUT / 1000}s, opacity ${DESTROY_FADEOUT / 1000}s linear;
+ transition: visibility 0s ${DESTROY_FADEOUT_MILLIS / 1000}s, opacity ${DESTROY_FADEOUT_MILLIS / 1000}s linear;
}
.hidden {
display: none;
@@ -275,7 +275,7 @@ export const layout = (url) => {
<div id="page-detail" class="hidden"></div>
<script>
const ws = new WebSocket("ws://${url}");
- const DESTROY_FADEOUT = ${DESTROY_FADEOUT};
+ const DESTROY_FADEOUT = ${DESTROY_FADEOUT_MILLIS};
const createPage = ${createPage.toString()};
const createPageCollection = ${createPageCollection.toString()};
const createBrowser = ${createBrowser.toString()};
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
index 2060f1562..b94af5f8e 100644
--- a/src/puppeteer_live_view_server.js
+++ b/src/puppeteer_live_view_server.js
@@ -8,7 +8,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import { layout, indexPage, detailPage, errorPage } from './puppeteer_live_view_client';
import { ENV_VARS } from './constants';
-const DEFAULT_SCREENSHOT_TIMEOUT = 3000;
+const DEFAULT_SCREENSHOT_TIMEOUT_MILLIS = 3000;
/**
* LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides
@@ -17,28 +17,28 @@ const DEFAULT_SCREENSHOT_TIMEOUT = 3000;
*
* @param {Browser} browser A Puppeteer Browser instance.
* @param {String} [options.id] A unique ID of the LiveViewBrowser.
- * @param {Number} [options.screenshotTimeout] Max time allowed for the screenshot taking process.
+ * @param {Number} [options.screenshotTimeoutMillis] Max time allowed for the screenshot taking process.
*/
export class PuppeteerLiveViewBrowser extends EventEmitter {
constructor(browser, opts = {}) {
super();
this.browser = browser;
- this.pages = new Map(); // to track all pages and their creation order for listing
- this.pageIds = new WeakMap(); // to avoid iteration over pages
- this.loadedPages = new WeakSet(); // to just track loaded state
+ this.pages = new Map(); // To track all pages and their creation order for listing
+ this.pageIds = new WeakMap(); // To avoid iteration over pages
+ this.loadedPages = new WeakSet(); // To just track loaded state
this.pageIdCounter = 0;
checkParamOrThrow(opts.id, 'opts.id', 'String');
- checkParamOrThrow(opts.screenshotTimeout, 'opts.screenshotTimeout', 'Maybe Number');
+ checkParamOrThrow(opts.screenshotTimeoutMillis, 'opts.screenshotTimeoutMillis', 'Maybe Number');
this.id = opts.id;
- this.screenshotTimeout = opts.screenshotTimeout || DEFAULT_SCREENSHOT_TIMEOUT;
+ this.screenshotTimeoutMillis = opts.screenshotTimeoutMillis || DEFAULT_SCREENSHOT_TIMEOUT_MILLIS;
- // since the page can be in any state when the user requests
- // a screenshot, we need to keep track of it ourselves
+ // Since the page can be in any state when the user requests
+ // a screenshot, we need to keep track of it ourselves.
browser.on('targetcreated', this._onTargetCreated.bind(this));
- // clean up resources after page close
+ // Clean up resources after page close
browser.on('targetdestroyed', this._onTargetDestroyed.bind(this));
- // clean up resources after browser close (in Server)
+ // Clean up resources after browser close (in Server)
browser.on('disconnected', this._onBrowserDisconnected.bind(this));
}
@@ -76,9 +76,9 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
log.error(err);
});
};
- // capture immediately for loaded pages
+ // Capture immediately for loaded pages
if (this.loadedPages.has(page)) capture();
- // setup recurrent capturing
+ // Setup recurrent capturing
page.on('load', () => {
if (this.listenerCount(id)) capture();
});
@@ -113,8 +113,8 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
* @ignore
*/
_getScreenshotAndHtml(page) {
- // replace page's close function to prevent a close
- // while the screenshot is being taken
+ // Replace page's close function to prevent a close
+ // while the screenshot is being taken.
const { close } = page;
let closed;
let closeArgs;
@@ -128,8 +128,8 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
};
const cleanup = () => {
- // replace the stolen close() method or call it,
- // if it should've been called externally
+ // Replace the stolen close() method or call it,
+ // if it should've been called externally.
if (closed) {
close.apply(page, closeArgs)
.then(closeResolve);
@@ -142,7 +142,7 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
image: page.screenshot(),
html: page.content(),
})
- .timeout(this.screenshotTimeout, 'Puppeteer Live View: Screenshot timed out.')
+ .timeout(this.screenshotTimeoutMillis, 'Puppeteer Live View: Screenshot timed out.')
.finally(cleanup);
}
@@ -152,33 +152,31 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
* @private
*/
_onTargetCreated(target) {
- if (target.type() === 'page') {
- target.page()
- .then((page) => {
- const id = `${this.id}_PAGE_${++this.pageIdCounter}`;
- this.pages.set(id, page);
- this.pageIds.set(page, id);
- this.emit('pagecreated', {
+ if (target.type() !== 'page') return;
+ target.page()
+ .then((page) => {
+ const id = `${this.id}_PAGE_${++this.pageIdCounter}`;
+ this.pages.set(id, page);
+ this.pageIds.set(page, id);
+ this.emit('pagecreated', {
+ id,
+ browserId: this.id,
+ url: page.url(),
+ });
+ page.on('load', () => {
+ this.loadedPages.add(page); // Page is loaded
+ });
+ // Using "framenavigated" on main Frame since "targetchanged" does not seem reliable
+ page.on('framenavigated', (frame) => {
+ if (frame !== page.mainFrame()) return;
+ this.loadedPages.delete(page);
+ this.emit('pagenavigated', {
id,
- browserId: this.id,
- url: page.url(),
+ url: frame.url(),
});
- page.on('load', () => {
- this.loadedPages.add(page); // page is loaded
- });
- // using "framenavigated" on main Frame since "targetchanged" does not seem reliable
- page.on('framenavigated', (frame) => {
- if (frame === page.mainFrame()) {
- this.loadedPages.delete(page); // page will load after nav
- this.emit('pagenavigated', {
- id,
- url: frame.url(),
- });
- }
- });
- })
- .catch(err => log.error(err));
- }
+ });
+ })
+ .catch(err => log.error(err));
}
/**
@@ -187,17 +185,16 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
* @private
*/
_onTargetDestroyed(target) {
- if (target.type() === 'page') {
- target.page()
- .then((page) => {
- const id = this.pageIds.get(page);
- this.pages.delete(id);
- this.emit('pagedestroyed', {
- id,
- });
- })
- .catch(err => log.error(err));
- }
+ if (target.type() !== 'page') return;
+ target.page()
+ .then((page) => {
+ const id = this.pageIds.get(page);
+ this.pages.delete(id);
+ this.emit('pagedestroyed', {
+ id,
+ });
+ })
+ .catch(err => log.error(err));
}
/**
* Handler invoked whenever a Browser is destroyed / disconnected in Puppeteer.
@@ -225,7 +222,7 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {
* @param {Promise<Browser>} browserPromise A Promise for a Puppeteer's Browser.
* @param {Object} [opts] Options to pass down to PuppeteerLiveViewBrowser constructor.
* @param {String} [opts.id] Custom ID to be used with the browser instance.
- * @param {String} [opts.screenshotTimeout] Max time allowed for the screenshot taking process.
+ * @param {String} [opts.screenshotTimeoutMillis] Max time allowed for the screenshot taking process.
* @returns {Promise<PuppeteerLiveViewServer>}
*/
let liveViewServer;
@@ -245,7 +242,7 @@ export const registerBrowserForLiveView = (browserPromise, opts = {}) => {
return liveViewServer;
});
- return Promise.all([serverPromise, browserPromise]).then(() => liveViewServer); // return PuppeteerLiveViewServer instance
+ return Promise.all([serverPromise, browserPromise]).then(() => liveViewServer); // Return PuppeteerLiveViewServer instance
};
/**
@@ -369,7 +366,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
status: 404,
};
- // traverses browsers to find a page with ID
+ // Traverses browsers to find a page with ID
const findPage = (id) => {
let page;
let browser;
@@ -385,7 +382,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
const COMMANDS = {
renderIndex: () => sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) }),
- // when the user selects a Page from the Index
+ // When the user selects a Page from the Index
renderPage: (msg) => {
const { id } = msg.data || {};
const [browser, page] = findPage(id);
@@ -396,7 +393,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
else sendCommand(ws, 'renderPage', { html: detailPage(pageData) });
});
},
- // when the user clicks on the back button
+ // When the user clicks on the back button
quitPage: (msg) => {
const { id } = msg.data || {};
const [browser, page] = findPage(id);
@@ -412,14 +409,14 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
} catch (err) {
return sendCommand(ws, 'error', BAD_REQUEST);
}
- // validate command and send response
+ // Validate command and send response
const { command } = msg;
const fn = COMMANDS[command];
if (!fn || typeof fn !== 'function') return sendCommand(ws, 'error', BAD_REQUEST);
fn(msg);
});
this._setupCommandHandles(ws);
- // send first Index after WS connection is established
+ // Send first Index after WS connection is established
sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });
}
@@ -438,7 +435,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
browser.on('pagecreated', createListener);
browser.on('pagedestroyed', destroyListener);
browser.on('pagenavigated', updateListener);
- // clean up to prevent listeners firing into closed sockets (on page refresh)
+ // Clean up to prevent listeners firing into closed sockets (on page refresh)
ws.on('close', () => {
browser.removeListener('pagecreated', createListener);
browser.removeListener('pagedestroyed', destroyListener);
@@ -458,7 +455,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
};
this.on('browsercreated', createListener);
this.on('browserdestroyed', destroyListener);
- // clean up to prevent listeners firing into closed sockets (on page refresh)
+ // Clean up to prevent listeners firing into closed sockets (on page refresh)
ws.on('close', () => {
this.removeListener('browsercreated', createListener);
this.removeListener('browserdestroyed', destroyListener);