Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

add missing client commands, clean up code a little

Ondrej Urban4766caa7e8fa
diff --git a/examples/synchronous_run.js b/examples/synchronous_run.js
index 731e703df..dcb3d3ab2 100644
--- a/examples/synchronous_run.js
+++ b/examples/synchronous_run.js
@@ -25,7 +25,7 @@ Apify.main(async () => {
const imageElementHandle = await iframe.$('.VideoColm img');
// Get a screenshot of that image.
- const imageBuffer = await imageElementHandle.screenshot();
+ const imageBuffer = await imageElementHandle.getScreenshotAndHtml();
// Save it as an output.
await Apify.setValue('OUTPUT', imageBuffer, { contentType: 'image/jpeg' });
diff --git a/src/puppeteer_live_view_browser.js b/src/puppeteer_live_view_browser.js
deleted file mode 100644
index 7b9fcfc4e..000000000
--- a/src/puppeteer_live_view_browser.js
+++ /dev/null
@@ -1,152 +0,0 @@
-import EventEmitter from 'events';
-import Promise from 'bluebird';
-import uuid from 'uuid/v4';
-import log from 'apify-shared/log';
-import { checkParamOrThrow } from 'apify-client/build/utils';
-import { imgPage, errorPage } from './puppeteer_live_view_client';
-import { createTimeoutPromise } from './utils';
-
-const DEFAULT_SCREENSHOT_TIMEOUT = 3000;
-
-/**
- * LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides
- * an API to safely get screenshots of the Browser's Pages.
- * @param {Browser} browser A Puppeteer Browser instance.
- * @param {String} [options.id] A unique ID of the LiveViewBrowser.
- * @param {Number} [options.screenshotTimeout] Max time allowed for the screenshot taking process.
- */
-export default class LiveViewBrowser extends EventEmitter {
- constructor(browser, opts = {}) {
- super();
- this.browser = browser;
- this.pages = new Map(); // to track all pages and their creation order for listing
- this._pageIds = new Map(); // to avoid iteration over pages
- this._loadedPages = new WeakSet(); // to just track loaded state
-
- checkParamOrThrow(opts.id, 'opts.id', 'Maybe String');
- checkParamOrThrow(opts.screenshotTimeout, 'opts.screenshotTimeout', 'Maybe Number');
- this.id = opts.id || uuid();
- this.screenshotTimeout = opts.screenshotTimeout || DEFAULT_SCREENSHOT_TIMEOUT;
-
- // since the page can be in any state when the user requests
- // a screenshot, we need to keep track of it ourselves
- browser.on('targetcreated', (target) => {
- if (target.type() === 'page') {
- target.page()
- .then((page) => {
- const id = uuid();
- this.pages.set(id, page);
- this._pageIds.set(page, id);
- this.emit('pagecreated', {
- id,
- browserId: this.id,
- url: page.url(),
- });
- page.on('load', () => {
- this._loadedPages.add(page); // page is loaded
- });
- page.on('framenavigated', (frame) => {
- this.emit('pagenavigated', {
- id,
- url: frame.url(),
- });
- });
- })
- .catch(err => log.error(err));
- }
- });
- browser.on('targetdestroyed', (target) => {
- if (target.type() === 'page') {
- target.page()
- .then((page) => {
- const id = this._pageIds.get(page);
- this.pages.delete(id);
- this.emit('pagedestroyed', {
- id,
- });
- })
- .catch(err => log.error(err));
- }
- });
- browser.on('disconnected', b => this.emit('disconnected', b));
- }
-
- /**
- * Handler that gets invoked by LiveViewRouter and sends an appropriate
- * response.
- * @param {http.IncomingMessage} req
- * @param {http.ServerResponse} res
- */
- routeHandler(req, res) {
- this.browser.pages()
- .then((pages) => {
- if (pages[pages.length - 1]) {
- return this._screenshot(pages[pages.length - 1]);
- }
- })
- .then((shot) => {
- dispatcher(res, imgPage(shot));
- })
- .catch((err) => {
- dispatcher(res, errorPage(err.message), 500);
- });
- }
-
- /**
- * The screenshot method simply takes a screenshot of the provided
- * Page and returns it as a promise. Unfortunately, nothing prevents
- * the Page from being closed while the screenshot is being taken,
- * which results into error. Therefore, the method prevents the page
- * from being closed by replacing its close method and handling the
- * page close itself once the screenshot has been taken.
- * @param {Page} page Puppeteer's Page
- * @returns {Promise<Buffer>} screenshot
- * @private
- */
- screenshot(page) {
- // replace page's close function to prevent a close
- // while the screenshot is being taken
- const { close } = page;
- let closed;
- let closeArgs;
- let closeResolve;
- page.close = (...args) => {
- if (!closed) closeArgs = args;
- closed = true;
- return new Promise((resolve) => {
- closeResolve = resolve;
- });
- };
-
- // check if the page has been marked as loaded
- const loaded = this._loadedPages.has(page);
-
- // setup promises
- const timeoutPromise = createTimeoutPromise(this.screenshotTimeout, 'Puppeteer Live View: Screenshot timed out.');
- const onLoadPromise = new Promise((resolve) => {
- page.on('load', () => {
- resolve(Promise.race([page.screenshot(), timeoutPromise]));
- });
- });
-
- // if page is already loaded, take a screenshot
- // otherwise, wait for it to load
- const result = loaded
- ? Promise.race([page.screenshot(), timeoutPromise])
- : onLoadPromise;
-
- result.finally(() => {
- // replace the stolen close() method or call it,
- // if it should've been called externally
- if (closed) {
- close.apply(page, closeArgs)
- .then(closeResolve);
- } else {
- page.close = close.bind(page);
- }
- });
-
- return result;
- }
-}
-
diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.js
index def3579ae..4f28752ec 100644
--- a/src/puppeteer_live_view_client.js
+++ b/src/puppeteer_live_view_client.js
@@ -1,6 +1,6 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
-const DESTROY_PAGE_FADEOUT = 2000;
+const DESTROY_FADEOUT = 2000;
const createPage = (id, url) => `<div class="page" id="${id}">URL: ${url}</div>`;
@@ -14,15 +14,21 @@ const createPageCollection = (pages) => {
const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>${id}</h3>${createPageCollection(pages)}</div>`;
+const createBrowserCollection = (browsers) => {
+ const browserDivs = [];
+ browsers.forEach(({ id, pages }) => {
+ browserDivs.push(createBrowser(id, pages));
+ });
+ return `<div class="browser-collection">${browserDivs.join('\n')}</div>`;
+};
+
/**
* Generates a body for the Index page. A list of browsers and their pages.
* @param {Set} browsers
* @returns {string} index page HTML
*/
export const indexPage = (browsers) => {
- const browserDivs = [];
- browsers.forEach(({ id, pages }) => browserDivs.push(createBrowser(id, pages)));
- return browserDivs.join('\n');
+ return createBrowserCollection(browsers);
};
/**
@@ -30,40 +36,28 @@ export const indexPage = (browsers) => {
* @param {Buffer} imageBuffer
* @returns {string}
*/
-export const detailPage = ({ id, url, image }) => {
+export const detailPage = ({ id, url, image, html }) => {
+ const chars = { '<': '&lt', '>': '&gt', '&': '&amp' };
+ const escapedHtml = html.replace(/[<>&]/g, m => chars[m]);
+
return `
<div id="${id}">
<h3>Detail of Page: ${url}</h3>
<div class="screenshot">
<img src="data:image/png;base64, ${image.toString('base64')}" alt="Page screenshot" />
- </div>
+ </div>
+ <pre>
+ <code class="original-html">
+ ${escapedHtml}
+ </code>
+ </pre>
+ </script>
+ </pre>
</div>
`;
};
-/**
- * Returns a 404 response page.
- * @returns {string}
- */
-export const notFoundPage = () => {
- const body = '<p>This page does not exist.</p>';
- return layout({ body });
-};
-
-/**
- * Returns an error page with an error message.
- * @param {String} message
- * @returns {string}
- */
-export const errorPage = (message) => {
- const body = `
-<p>Sorry. There was an error and Live View failed.</p>
-${message ? `<p>Message: ${message}</p>` : ''}
-`;
- return layout({ body });
-};
-
const wsHandler = (socket) => {
const index = document.getElementById('index');
const pageDetail = document.getElementById('page-detail');
@@ -96,6 +90,16 @@ const wsHandler = (socket) => {
pageDetail.classList.remove('hidden');
pageDetail.innerHTML = html;
},
+ createBrowser: ({ id }) => {
+ const browsers = index.querySelector('.browser-collection');
+ browsers.insertAdjacentHTML('afterbegin', createBrowser(id));
+ },
+ destroyBrowser: ({ id }) => {
+ const browser = document.getElementById(id);
+ browser.classList.add('destroyed');
+ // do not remove immediately
+ setTimeout(() => browser.remove(), DESTROY_FADEOUT);
+ },
createPage: ({ id, browserId, url }) => {
const pages = document.getElementById(browserId).querySelector('.page-collection');
pages.insertAdjacentHTML('afterbegin', createPage(id, url));
@@ -113,7 +117,7 @@ const wsHandler = (socket) => {
page.classList.add('destroyed');
// do not remove immediately since it can happen pretty fast
// and the page only pops in the list for a split second
- setTimeout(() => page.remove(), DESTROY_PAGE_FADEOUT);
+ setTimeout(() => page.remove(), DESTROY_FADEOUT);
},
error: ({ message, status }) => {
console.error(`${status}: ${message}`); // eslint-disable-line
@@ -169,11 +173,14 @@ export const layout = (opts = {}) => {
color: #660000;
visibility: hidden;
opacity: 0;
- transition: visibility 0s 2s, opacity 2s linear;
+ transition: visibility 0s ${DESTROY_FADEOUT / 1000}s, opacity ${DESTROY_FADEOUT / 1000}s linear;
}
.hidden {
display: none;
}
+ .original-html {
+ white-space: pre-wrap;
+ }
</style>
</head>
<body>
@@ -183,7 +190,7 @@ export const layout = (opts = {}) => {
<div id="page-detail" class="hidden"></div>
<script>
const ws = new WebSocket("ws://${opts.host}:${opts.port}");
- const DESTROY_PAGE_FADEOUT = ${DESTROY_PAGE_FADEOUT};
+ const DESTROY_FADEOUT = ${DESTROY_FADEOUT};
const createPage = ${createPage.toString()};
const createPageCollection = ${createPageCollection.toString()};
const createBrowser = ${createBrowser.toString()};
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
index 70e863023..d53cab33e 100644
--- a/src/puppeteer_live_view_server.js
+++ b/src/puppeteer_live_view_server.js
@@ -1,15 +1,141 @@
import http from 'http';
import EventEmitter from 'events';
+import Promise from 'bluebird';
+import uuid from 'uuid/v4';
+import WebSocket from 'ws';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import WebSocket from 'ws';
-import LiveViewBrowser from './puppeteer_live_view_browser';
+import { createTimeoutPromise } from './utils';
import { layout, indexPage, detailPage } from './puppeteer_live_view_client';
const LOCAL_IPV6 = '::';
const LOCAL_IPV4 = '127.0.0.1';
+const DEFAULT_SCREENSHOT_TIMEOUT = 3000;
+
+/**
+ * LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides
+ * an API to safely get screenshots of the Browser's Pages.
+ * @param {Browser} browser A Puppeteer Browser instance.
+ * @param {String} [options.id] A unique ID of the LiveViewBrowser.
+ * @param {Number} [options.screenshotTimeout] Max time allowed for the screenshot taking process.
+ */
+export class LiveViewBrowser extends EventEmitter {
+ constructor(browser, opts = {}) {
+ super();
+ this.browser = browser;
+ this.pages = new Map(); // to track all pages and their creation order for listing
+ this._pageIds = new WeakMap(); // to avoid iteration over pages
+ this._loadedPages = new WeakSet(); // to just track loaded state
+
+ checkParamOrThrow(opts.id, 'opts.id', 'Maybe String');
+ checkParamOrThrow(opts.screenshotTimeout, 'opts.screenshotTimeout', 'Maybe Number');
+ this.id = opts.id || uuid();
+ this.screenshotTimeout = opts.screenshotTimeout || DEFAULT_SCREENSHOT_TIMEOUT;
+
+ // since the page can be in any state when the user requests
+ // a screenshot, we need to keep track of it ourselves
+ browser.on('targetcreated', (target) => {
+ if (target.type() === 'page') {
+ target.page()
+ .then((page) => {
+ const id = uuid();
+ this.pages.set(id, page);
+ this._pageIds.set(page, id);
+ this.emit('pagecreated', {
+ id,
+ browserId: this.id,
+ url: page.url(),
+ });
+ page.on('load', () => {
+ this._loadedPages.add(page); // page is loaded
+ });
+ page.on('framenavigated', (frame) => {
+ this.emit('pagenavigated', {
+ id,
+ url: frame.url(),
+ });
+ });
+ })
+ .catch(err => log.error(err));
+ }
+ });
+ browser.on('targetdestroyed', (target) => {
+ if (target.type() === 'page') {
+ target.page()
+ .then((page) => {
+ const id = this._pageIds.get(page);
+ this.pages.delete(id);
+ this.emit('pagedestroyed', {
+ id,
+ });
+ })
+ .catch(err => log.error(err));
+ }
+ });
+ browser.on('disconnected', b => this.emit('disconnected', b));
+ }
+
+ /**
+ * The getScreenshotAndHtml method simply retrieves the page's HTML
+ * content, takes a screenshot and returns both as a promise.
+ * Unfortunately, nothing prevents the Page from being closed while
+ * the screenshot is being taken, which results into error.
+ * Therefore, the method prevents the page from being closed
+ * by replacing its close method and handling the page close
+ * itself once the screenshot has been taken.
+ * @param {Page} page Puppeteer's Page
+ * @returns {Promise<Buffer>} screenshot
+ * @private
+ */
+ getScreenshotAndHtml(page) {
+ // replace page's close function to prevent a close
+ // while the screenshot is being taken
+ const { close } = page;
+ let closed;
+ let closeArgs;
+ let closeResolve;
+ page.close = (...args) => {
+ if (!closed) closeArgs = args;
+ closed = true;
+ return new Promise((resolve) => {
+ closeResolve = resolve;
+ });
+ };
+
+ // setup promise factories
+ const data = () => new Promise((resolve, reject) => {
+ const result = {};
+ const invoke = () => {
+ const image = page.screenshot().then((s) => { result.image = s; });
+ const html = page.content().then((s) => { result.html = s; });
+ Promise.all([image, html])
+ .then(() => resolve(result))
+ .catch(reject);
+ };
+ // if page is already loaded, take a screenshot
+ // otherwise, wait for it to load
+ if (this._loadedPages.has(page)) {
+ invoke();
+ } else {
+ page.on('load', invoke);
+ }
+ });
+ const timeout = () => createTimeoutPromise(this.screenshotTimeout, 'Puppeteer Live View: Screenshot timed out.');
+ const cleanup = () => {
+ // replace the stolen close() method or call it,
+ // if it should've been called externally
+ if (closed) {
+ close.apply(page, closeArgs)
+ .then(closeResolve);
+ } else {
+ page.close = close.bind(page);
+ }
+ };
+
+ return Promise.race([data(), timeout()]).finally(cleanup);
+ }
+}
-let defaultServer;
/**
* The start method should cover most use cases of starting a PuppeteerLiveViewServer.
@@ -27,6 +153,7 @@ let defaultServer;
* @param {String} [opts.screenshotTimeout] Max time allowed for the screenshot taking process.
* @returns {Promise<PuppeteerLiveViewServer>} The promise will resolve when the promise for Puppeteer's Browser resolves.
*/
+let defaultServer;
export const startPuppeteerLiveView = (browserPromise, opts = {}) => {
if (!defaultServer) {
defaultServer = new PuppeteerLiveViewServer(opts);
@@ -148,12 +275,13 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
}
}
if (!page) return sendCommand(ws, 'error', NOT_FOUND);
- browser.screenshot(page)
- .then((image) => {
+ browser.getScreenshotAndHtml(page)
+ .then(({ image, html }) => {
const data = {
id,
url: page.url(),
image,
+ html,
};
sendCommand(ws, 'renderPage', { html: detailPage(data) });
})
@@ -162,11 +290,11 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
sendCommand(ws, 'error', BAD_REQUEST);
}
});
+ this._setupCommandHandles(ws);
sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });
- this._wsPushData(ws);
}
- _wsPushData(ws) {
+ _setupCommandHandles(ws) {
const attachListeners = (browser) => {
const createListener = p => sendCommand(ws, 'createPage', p);
const destroyListener = p => sendCommand(ws, 'destroyPage', p);