Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
implement client side websocket handling
Ondrej Urbanb9e29e5bbeed
diff --git a/src/live_view_html.js b/src/live_view_html.jsdeleted file mode 100644index 8604b6831..000000000--- a/src/live_view_html.js+++ /dev/null@@ -1,100 +0,0 @@-import { checkParamOrThrow } from 'apify-client/build/utils';--/**- * Template for a basic layout of a HTML page.- * @param {String} opts.host hostname of the WebSocket server- * @param {Number} opts.port port of the WebSocket server- * @returns {string} html- */-export const layout = (opts = {}) => {- checkParamOrThrow(opts.host, 'opts.host', 'String');- checkParamOrThrow(opts.port, 'opts.port', 'Number');-- return `-<!doctype html>-<html>-<head>- <meta charset="utf-8">- <title>Live View Server</title>- <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">- ${opts.refresh ? `<meta http-equiv="refresh" content="${opts.refresh}">` : ''}-</head>-<body>- <div id="ws-container">Waiting for WebSocket connection.</div>- <script>- var container = document.getElementById("ws-container");- var ws = new WebSocket("ws://${opts.host}:${opts.port}");- ws.onmessage = (e) => {- container.innerHTML = e.data;- }- </script>-</body>-</html>-`;-};--/**- * Generates a body for the rootPage. A list of browsers and their IDs.- * @param {Array} browsers- * @returns {string}- */-export const rootPage = (browsers) => {- const body = `-<h1>Available live view browsers:</h1>- ${browsers.map(b => `- <a href="/browser/${b.id}"><p>ID: ${b.id}</p></a>- `).join('')}-`;- return layout({- body,- refresh: 3,- });-};--/**- * Returns an image encoded as a base64 string in an <img> tag.- * @param {Buffer} imageBuffer- * @returns {string}- */-export const encodeImg = (imageBuffer) => {- return `-<div>- <img src="data:image/png;base64, ${imageBuffer.toString('base64')}" alt="Page screenshot" />-</div>-`;-};--/**- * Returns a body of a page consisting of a serialized image.- * @param {Buffer} imageBuffer- * @returns {string}- */-export const imgPage = (imageBuffer) => {- const body = encodeImg(imageBuffer);- return layout({- body,- refresh: 1,- });-};--/**- * Returns a 404 response page.- * @returns {string}- */-export const notFoundPage = () => {- const body = '<p>This page does not exist.</p>';- return layout({ body });-};--/**- * Returns an error page with an error message.- * @param {String} message- * @returns {string}- */-export const errorPage = (message) => {- const body = `-<p>Sorry. There was an error and Live View failed.</p>-${message ? `<p>Message: ${message}</p>` : ''}-`;- return layout({ body });-};diff --git a/src/live_view_router.js b/src/live_view_router.jsdeleted file mode 100644index 596d84710..000000000--- a/src/live_view_router.js+++ /dev/null@@ -1,63 +0,0 @@-import { checkParamOrThrow } from 'apify-client/build/utils';-import { notFoundPage, rootPage } from './live_view_html';--/**- * Initiates server response with predefined Content-Type and Content-Length headers.- * @param {http.ServerResponse} res- * @param {String | Buffer} body- * @param {Number} [status]- */-export const dispatcher = (res, body, status) => {- checkParamOrThrow(res, 'res', 'Object');- checkParamOrThrow(body, 'body', 'Maybe String | Buffer');- checkParamOrThrow(status, 'opts.status', 'Maybe Number');- res.writeHead(status || 200, {- 'Content-Type': 'text/html',- 'Content-Length': Buffer.byteLength(body),- });- res.end(body);-};--/**- * Individual route handlers that use HTML templates to serialize responses.- * @param {http.IncomingMessage} req- * @param {http.ServerResponse} res- */-const rootPageHandler = (req, res) => dispatcher(res, rootPage(req.lvs.browsers));-const notFoundHandler = (req, res) => dispatcher(res, notFoundPage(), 404);--/**- * LiveViewRouter class encapsulates routing logic of the PuppeteerLiveViewServer.- */-export default class LiveViewRouter {- constructor() {- this.routes = {- '': rootPageHandler,- };- }-- /**- * Creates a route that points to the given LiveViewBrowser by its id.- * @param {LiveViewBrowser} browser- */- addBrowser(browser) {- checkParamOrThrow(browser.id, 'browser.id', 'String');- checkParamOrThrow(browser.routeHandler, 'browser.routeHandler', 'Function');- const route = `browser/${browser.id}`;- this.routes[route] = browser.routeHandler.bind(browser);- }-- /**- * The main route handling function of LiveViewRouter. Attempts to invoke a specific route- * handler. If none is found, invokes 404 handler.- * @param {String} path- * @param {http.IncomingMessage} req- * @param {http.ServerResponse} res- */- handle(path, req, res) {- const handler = this.routes[path];- if (handler) return handler(req, res);- notFoundHandler(req, res);- }-}-diff --git a/src/puppeteer.js b/src/puppeteer.jsindex b34b80220..5e93519d6 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -5,7 +5,7 @@ import log from 'apify-shared/log';import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';import { newPromise, getTypicalChromeExecutablePath } from './utils';import { getApifyProxyUrl } from './actor';-import { startPuppeteerLiveView } from './live_view_server';+import { startPuppeteerLiveView } from './puppeteer_live_view_server';/* global process, require */diff --git a/src/live_view_browser.js b/src/puppeteer_live_view_browser.jssimilarity index 78%rename from src/live_view_browser.jsrename to src/puppeteer_live_view_browser.jsindex fe80a031f..7b9fcfc4e 100644--- a/src/live_view_browser.js+++ b/src/puppeteer_live_view_browser.js@@ -3,11 +3,10 @@ import Promise from 'bluebird';import uuid from 'uuid/v4';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';-import { dispatcher } from './live_view_router';-import { imgPage, errorPage } from './live_view_html';+import { imgPage, errorPage } from './puppeteer_live_view_client';import { createTimeoutPromise } from './utils';-const DEFAULT_SCREENSHOT_TIMEOUT = 1000;+const DEFAULT_SCREENSHOT_TIMEOUT = 3000;/*** LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides@@ -21,7 +20,8 @@ export default class LiveViewBrowser extends EventEmitter {super();this.browser = browser;this.pages = new Map(); // to track all pages and their creation order for listing- this.loadedPages = new WeakSet(); // to just track loaded state+ this._pageIds = new Map(); // to avoid iteration over pages+ this._loadedPages = new WeakSet(); // to just track loaded statecheckParamOrThrow(opts.id, 'opts.id', 'Maybe String');checkParamOrThrow(opts.screenshotTimeout, 'opts.screenshotTimeout', 'Maybe Number');@@ -34,10 +34,22 @@ export default class LiveViewBrowser extends EventEmitter {if (target.type() === 'page') {target.page().then((page) => {- this.pages.set(uuid(), page);- this.emit('pagecreated', page);+ const id = uuid();+ this.pages.set(id, page);+ this._pageIds.set(page, id);+ this.emit('pagecreated', {+ id,+ browserId: this.id,+ url: page.url(),+ });page.on('load', () => {- this.loadedPages.add(page); // page is loaded+ this._loadedPages.add(page); // page is loaded+ });+ page.on('framenavigated', (frame) => {+ this.emit('pagenavigated', {+ id,+ url: frame.url(),+ });});}).catch(err => log.error(err));@@ -47,17 +59,11 @@ export default class LiveViewBrowser extends EventEmitter {if (target.type() === 'page') {target.page().then((page) => {- this.pages.delete(page);- this.emit('pagedestroyed', page);- })- .catch(err => log.error(err));- }- });- browser.on('targetchanged', (target) => {- if (target.type() === 'page') {- target.page()- .then((page) => {- this.emit('pagechanged', page);+ const id = this._pageIds.get(page);+ this.pages.delete(id);+ this.emit('pagedestroyed', {+ id,+ });}).catch(err => log.error(err));}@@ -97,7 +103,7 @@ export default class LiveViewBrowser extends EventEmitter {* @returns {Promise<Buffer>} screenshot* @private*/- _screenshot(page) {+ screenshot(page) {// replace page's close function to prevent a close// while the screenshot is being takenconst { close } = page;@@ -113,7 +119,7 @@ export default class LiveViewBrowser extends EventEmitter {};// check if the page has been marked as loaded- const loaded = this.loadedPages.has(page);+ const loaded = this._loadedPages.has(page);// setup promisesconst timeoutPromise = createTimeoutPromise(this.screenshotTimeout, 'Puppeteer Live View: Screenshot timed out.');diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.jsnew file mode 100644index 000000000..def3579ae--- /dev/null+++ b/src/puppeteer_live_view_client.js@@ -0,0 +1,195 @@+import { checkParamOrThrow } from 'apify-client/build/utils';++const DESTROY_PAGE_FADEOUT = 2000;++const createPage = (id, url) => `<div class="page" id="${id}">URL: ${url}</div>`;++const createPageCollection = (pages) => {+ const pageDivs = [];+ pages.forEach((page, id) => {+ pageDivs.push(createPage(id, page.url()));+ });+ return `<div class="page-collection">${pageDivs.join('\n')}</div>`;+};++const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>${id}</h3>${createPageCollection(pages)}</div>`;++/**+ * Generates a body for the Index page. A list of browsers and their pages.+ * @param {Set} browsers+ * @returns {string} index page HTML+ */+export const indexPage = (browsers) => {+ const browserDivs = [];+ browsers.forEach(({ id, pages }) => browserDivs.push(createBrowser(id, pages)));+ return browserDivs.join('\n');+};++/**+ * Returns a body of a page consisting of a serialized image.+ * @param {Buffer} imageBuffer+ * @returns {string}+ */+export const detailPage = ({ id, url, image }) => {+ return `+<div id="${id}">+ <h3>Detail of Page: ${url}</h3>+ <div class="screenshot">+ <img src="data:image/png;base64, ${image.toString('base64')}" alt="Page screenshot" />+ </div>+</div>++`;+};++/**+ * Returns a 404 response page.+ * @returns {string}+ */+export const notFoundPage = () => {+ const body = '<p>This page does not exist.</p>';+ return layout({ body });+};++/**+ * Returns an error page with an error message.+ * @param {String} message+ * @returns {string}+ */+export const errorPage = (message) => {+ const body = `+<p>Sorry. There was an error and Live View failed.</p>+${message ? `<p>Message: ${message}</p>` : ''}+`;+ return layout({ body });+};++const wsHandler = (socket) => {+ const index = document.getElementById('index');+ const pageDetail = document.getElementById('page-detail');+ const backButton = document.getElementById('back-button');+ backButton.onclick = () => {+ pageDetail.classList.add('hidden');+ backButton.classList.add('hidden');+ index.classList.remove('hidden');+ };++ const sendCommand = (command, data) => {+ if (socket.readyState === 1) {+ const payload = JSON.stringify({ command, data });+ socket.send(payload);+ }+ };++ const COMMANDS = {+ renderIndex: ({ html }) => {+ index.innerHTML = html;+ index.querySelectorAll('.page').forEach((page) => {+ page.onclick = () => sendCommand('renderPage', {+ id: page.getAttribute('id'),+ });+ });+ },+ renderPage: ({ html }) => {+ index.classList.add('hidden');+ backButton.classList.remove('hidden');+ pageDetail.classList.remove('hidden');+ pageDetail.innerHTML = html;+ },+ createPage: ({ id, browserId, url }) => {+ const pages = document.getElementById(browserId).querySelector('.page-collection');+ pages.insertAdjacentHTML('afterbegin', createPage(id, url));+ const page = document.getElementById(id);+ page.onclick = () => sendCommand('renderPage', {+ id: page.getAttribute('id'),+ });+ },+ updatePage: ({ id, url }) => {+ const page = document.getElementById(id);+ page.innerText = `URL: ${url}`;+ },+ destroyPage: ({ id }) => {+ const page = document.getElementById(id);+ page.classList.add('destroyed');+ // do not remove immediately since it can happen pretty fast+ // and the page only pops in the list for a split second+ setTimeout(() => page.remove(), DESTROY_PAGE_FADEOUT);+ },+ error: ({ message, status }) => {+ console.error(`${status}: ${message}`); // eslint-disable-line+ },+ };++ socket.onmessage = (e) => {+ let message;+ try {+ message = JSON.parse(e.data);+ } catch (err) {+ return console.error('Unable to parse message from server:', e.data); // eslint-disable-line+ }+ const { command, data } = message;+ if (!command) return console.error('Invalid command:', command); // eslint-disable-line+ const fn = COMMANDS[command];+ if (!fn || typeof fn !== 'function') return console.error('Command not recognized by client:', command); // eslint-disable-line+ fn(data);+ };++ socket.onclose = () => {+ index.insertAdjacentHTML('beforebegin', '<div>Act finished</div>');+ };++ socket.onerror = (err) => {+ console.error(err); //eslint-disable-line+ };+};++/**+ * Template for a basic layout of a HTML page.+ * @param {String} opts.host hostname of the WebSocket server+ * @param {Number} opts.port port of the WebSocket server+ * @returns {string} html+ */+export const layout = (opts = {}) => {+ checkParamOrThrow(opts.host, 'opts.host', 'String');+ checkParamOrThrow(opts.port, 'opts.port', 'Number');++ return `+<!doctype html>+<html>+<head>+ <meta charset="utf-8">+ <title>Puppeteer Live View Server</title>+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">+ <style>+ .page:hover {+ cursor: pointer;+ background: lightgreen;+ }+ .destroyed {+ color: #660000;+ visibility: hidden;+ opacity: 0;+ transition: visibility 0s 2s, opacity 2s linear;+ }+ .hidden {+ display: none;+ }+ </style>+</head>+<body>+ <h1>Puppeteer Live View</h1>+ <button id="back-button" class="hidden">Back to Index</button>+ <div id="index">Waiting for WebSocket connection.</div>+ <div id="page-detail" class="hidden"></div>+ <script>+ const ws = new WebSocket("ws://${opts.host}:${opts.port}");+ const DESTROY_PAGE_FADEOUT = ${DESTROY_PAGE_FADEOUT};+ const createPage = ${createPage.toString()};+ const createPageCollection = ${createPageCollection.toString()};+ const createBrowser = ${createBrowser.toString()};+ (${wsHandler.toString()})(ws);+ </script>+</body>+</html>+`;+};diff --git a/src/live_view_server.js b/src/puppeteer_live_view_server.jssimilarity index 50%rename from src/live_view_server.jsrename to src/puppeteer_live_view_server.jsindex 95594e5b1..70e863023 100644--- a/src/live_view_server.js+++ b/src/puppeteer_live_view_server.js@@ -1,16 +1,11 @@import http from 'http';-import uuid from 'uuid/v4';+import EventEmitter from 'events';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import WebSocket from 'ws';-import LiveViewRouter from './live_view_router';-import LiveViewBrowser from './live_view_browser';-import { layout } from './live_view_html';+import LiveViewBrowser from './puppeteer_live_view_browser';+import { layout, indexPage, detailPage } from './puppeteer_live_view_client';-const BAD_REQUEST = {- error: 'Invalid WebSocket message.',- status: 400,-};const LOCAL_IPV6 = '::';const LOCAL_IPV4 = '127.0.0.1';@@ -45,48 +40,48 @@ export const startPuppeteerLiveView = (browserPromise, opts = {}) => {return browserPromise.then((browser) => {const lvb = new LiveViewBrowser(browser, browserOpts);- defaultServer.browsers.set(uuid(), lvb);+ defaultServer.addBrowser(lvb);return defaultServer;});};+const sendCommand = (socket, command, data) => {+ const payload = JSON.stringify({ command, data });+ socket.send(payload, (err) => {+ if (err) log.error(err);+ });+};+/*** Enables Live View monitoring of Act execution by spawning a web server that responds with a list* of available browsers at its root path. Once the user chooses a browser, PuppeteerLiveViewServer will* periodically serve screenshots of the selected browser's latest loaded page.* @param {Number} [opts.port] Listening port of the PuppeteerLiveViewServer. Defaults to 1234.*/-export default class PuppeteerLiveViewServer {+export default class PuppeteerLiveViewServer extends EventEmitter {constructor(opts = {}) {+ super();checkParamOrThrow(opts, 'opts', 'Object');checkParamOrThrow(opts.port, 'opts.port', 'Maybe String | Number');- this.browsers = new Map();+ this.browsers = new Set();this.port = Number(opts.port) || 1234;this.httpServer = null;}+ addBrowser(browser) {+ this.browsers.add(browser);+ this.emit('browsercreated', browser);+ browser.on('disconnected', () => this.deleteBrowser(browser));+ }- // static start(browserPromise, opts = {}) {- // if (!PuppeteerLiveViewServer.server) {- // PuppeteerLiveViewServer.server = new PuppeteerLiveViewServer(opts);- // PuppeteerLiveViewServer.server.startServer();- // }- // const lvs = PuppeteerLiveViewServer.server;- // const browserOpts = {- // id: opts.browserId,- // screenshotTimeout: opts.screenshotTimeout,- // };- //- // return browserPromise- // .then((browser) => {- // const lvb = new LiveViewBrowser(browser, browserOpts);- // lvs.browsers.push(lvb);- // lvs.router.addBrowser(lvb);- // });- // }+ deleteBrowser(browser) {+ this.browsers.delete(browser);+ this.emit('browserdestroyed', browser);+ browser.removeAllListeners();+ }/**- * Starts an HTTP server on a preconfigured port or 1234.+ * Starts an HTTP and a WebSocket server on a preconfigured port or 1234.*/startServer() {const server = http.createServer(this._httpRequestListener.bind(this));@@ -120,6 +115,16 @@ export default class PuppeteerLiveViewServer {}_wsRequestListener(ws) {+ const BAD_REQUEST = {+ message: 'Bad Request',+ status: 400,+ };+ const NOT_FOUND = {+ message: 'Not Found',+ status: 404,+ };++log.debug('WebSocket connection to Puppeteer Live View established.');ws.on('message', (msg) => {try {@@ -130,14 +135,67 @@ export default class PuppeteerLiveViewServer {const { command } = msg;if (command === 'renderIndex') {- ws.send('INDEX');+ sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });} else if (command === 'renderPage') {- const { id } = msg;- ws.send('PAGE');+ const { id } = msg.data;+ let page;+ let browser;+ for (const b of this.browsers) { // eslint-disable-line+ if (b.pages.has(id)) {+ page = b.pages.get(id);+ browser = b;+ break;+ }+ }+ if (!page) return sendCommand(ws, 'error', NOT_FOUND);+ browser.screenshot(page)+ .then((image) => {+ const data = {+ id,+ url: page.url(),+ image,+ };+ sendCommand(ws, 'renderPage', { html: detailPage(data) });+ })+ .catch(err => log.error(err));} else {- return ws.send(BAD_REQUEST);+ sendCommand(ws, 'error', BAD_REQUEST);}});- ws.send('INDEX');+ sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });+ this._wsPushData(ws);+ }++ _wsPushData(ws) {+ const attachListeners = (browser) => {+ const createListener = p => sendCommand(ws, 'createPage', p);+ const destroyListener = p => sendCommand(ws, 'destroyPage', p);+ const updateListener = p => sendCommand(ws, 'updatePage', p);+ browser.on('pagecreated', createListener);+ browser.on('pagedestroyed', destroyListener);+ browser.on('pagenavigated', updateListener);+ // clean up to prevent listeners firing into closed sockets (on page refresh)+ ws.on('close', () => {+ browser.removeListener('pagecreated', createListener);+ browser.removeListener('pagedestroyed', destroyListener);+ browser.removeListener('pagenavigated', updateListener);+ });+ };++ this.browsers.forEach(attachListeners);+ const createListener = (browser) => {+ attachListeners(browser);+ sendCommand(ws, 'createBrowser', { id: browser.id });+ };+ const destroyListener = (browser) => {+ sendCommand(ws, 'destroyBrowser', { id: browser.id });+ };+ this.on('browsercreated', createListener);+ this.on('browserdestroyed', destroyListener);+ // clean up to prevent listeners firing into closed sockets (on page refresh)+ ws.on('close', () => {+ this.removeListener('browsercreated', createListener);+ this.removeListener('browserdestroyed', destroyListener);+ });}}