Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Clean up old live view

Ondra Urban92b3c1ed40c2
diff --git a/package.json b/package.json
index ec122fc59..69668cee6 100644
--- a/package.json
+++ b/package.json
@@ -92,6 +92,7 @@
"proxy": "^0.2.4",
"sinon": "^7.1.1",
"sinon-stub-promise": "^4.0.0",
+ "socket.io-client": "^2.2.0",
"why-is-node-running": "^2.0.3"
},
"optionalDependencies": {
diff --git a/src/index.js b/src/index.js
index ec2d6feee..9bce6dbbb 100644
--- a/src/index.js
+++ b/src/index.js
@@ -21,6 +21,7 @@ import { puppeteerUtils } from './puppeteer_utils';
import { socialUtils } from './utils_social';
import { enqueueLinks } from './enqueue_links';
import PseudoUrl from './pseudo_url';
+import LiveViewServer from './live_view/live_view_server';
/* globals module */
@@ -101,6 +102,9 @@ module.exports = {
// Settings rotator
SettingsRotator,
+ // Live View
+ LiveViewServer,
+
// Webdriver
browse,
launchWebDriver,
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index a8a16b958..3de5c7670 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -92,6 +92,7 @@ export default class LiveViewServer {
}
async stop() {
+ this.httpServer.unref();
return new Promise((resolve) => {
this.httpServer.close((err) => {
this._isRunning = false;
diff --git a/src/live_view/prism.js b/src/live_view/prism.js
index f41b0c15f..668bab7d7 100644
--- a/src/live_view/prism.js
+++ b/src/live_view/prism.js
@@ -1,3 +1,4 @@
+/* eslint-disable */
/* PrismJS 1.15.0
https://prismjs.com/download.html#themes=prism&languages=markup */
var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(e){function t(e,t,a,n,r){this.type=e,this.content=t,this.alias=a,this.length=0|(n||"").length,this.greedy=!!r}var a=/\blang(?:uage)?-([\w-]+)\b/i,n=0,r={manual:e.Prism&&e.Prism.manual,disableWorkerMessageHandler:e.Prism&&e.Prism.disableWorkerMessageHandler,util:{encode:function(e){return e instanceof t?new t(e.type,r.util.encode(e.content),e.alias):"Array"===r.util.type(e)?e.map(r.util.encode):e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++n}),e.__id},clone:function l(e,t){var a,n,i=r.util.type(e);switch(t=t||{},i){case"Object":if(n=r.util.objId(e),t[n])return t[n];a={},t[n]=a;for(var o in e)e.hasOwnProperty(o)&&(a[o]=l(e[o],t));return a;case"Array":return n=r.util.objId(e),t[n]?t[n]:(a=[],t[n]=a,e.forEach(function(e,n){a[n]=l(e,t)}),a);default:return e}}},languages:{extend:function(e,t){var a=r.util.clone(r.languages[e]);for(var n in t)a[n]=t[n];return a},insertBefore:function(e,t,a,n){n=n||r.languages;var i=n[e],l={};for(var o in i)if(i.hasOwnProperty(o)){if(o==t)for(var s in a)a.hasOwnProperty(s)&&(l[s]=a[s]);a.hasOwnProperty(o)||(l[o]=i[o])}var u=n[e];return n[e]=l,r.languages.DFS(r.languages,function(t,a){a===u&&t!=e&&(this[t]=l)}),l},DFS:function o(e,t,a,n){n=n||{};var i=r.util.objId;for(var l in e)if(e.hasOwnProperty(l)){t.call(e,l,e[l],a||l);var s=e[l],u=r.util.type(s);"Object"!==u||n[i(s)]?"Array"!==u||n[i(s)]||(n[i(s)]=!0,o(s,t,l,n)):(n[i(s)]=!0,o(s,t,null,n))}}},plugins:{},highlightAll:function(e,t){r.highlightAllUnder(document,e,t)},highlightAllUnder:function(e,t,a){var n={callback:a,selector:'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code'};r.hooks.run("before-highlightall",n);for(var i,l=n.elements||e.querySelectorAll(n.selector),o=0;i=l[o++];)r.highlightElement(i,t===!0,n.callback)},highlightElement:function(t,n,i){for(var l,o,s=t;s&&!a.test(s.className);)s=s.parentNode;s&&(l=(s.className.match(a)||[,""])[1].toLowerCase(),o=r.languages[l]),t.className=t.className.replace(a,"").replace(/\s+/g," ")+" language-"+l,t.parentNode&&(s=t.parentNode,/pre/i.test(s.nodeName)&&(s.className=s.className.replace(a,"").replace(/\s+/g," ")+" language-"+l));var u=t.textContent,g={element:t,language:l,grammar:o,code:u},c=function(e){g.highlightedCode=e,r.hooks.run("before-insert",g),g.element.innerHTML=g.highlightedCode,r.hooks.run("after-highlight",g),r.hooks.run("complete",g),i&&i.call(g.element)};if(r.hooks.run("before-sanity-check",g),!g.code)return r.hooks.run("complete",g),void 0;if(r.hooks.run("before-highlight",g),!g.grammar)return c(r.util.encode(g.code)),void 0;if(n&&e.Worker){var h=new Worker(r.filename);h.onmessage=function(e){c(e.data)},h.postMessage(JSON.stringify({language:g.language,code:g.code,immediateClose:!0}))}else c(r.highlight(g.code,g.grammar,g.language))},highlight:function(e,a,n){var i={code:e,grammar:a,language:n};return r.hooks.run("before-tokenize",i),i.tokens=r.tokenize(i.code,i.grammar),r.hooks.run("after-tokenize",i),t.stringify(r.util.encode(i.tokens),i.language)},matchGrammar:function(e,a,n,i,l,o,s){for(var u in n)if(n.hasOwnProperty(u)&&n[u]){if(u==s)return;var g=n[u];g="Array"===r.util.type(g)?g:[g];for(var c=0;c<g.length;++c){var h=g[c],d=h.inside,f=!!h.lookbehind,p=!!h.greedy,m=0,y=h.alias;if(p&&!h.pattern.global){var v=h.pattern.toString().match(/[imuy]*$/)[0];h.pattern=RegExp(h.pattern.source,v+"g")}h=h.pattern||h;for(var k=i,b=l;k<a.length;b+=a[k].length,++k){var w=a[k];if(a.length>e.length)return;if(!(w instanceof t)){if(p&&k!=a.length-1){h.lastIndex=b;var A=h.exec(e);if(!A)break;for(var P=A.index+(f?A[1].length:0),O=A.index+A[0].length,x=k,N=b,j=a.length;j>x&&(O>N||!a[x].type&&!a[x-1].greedy);++x)N+=a[x].length,P>=N&&(++k,b=N);if(a[k]instanceof t)continue;S=x-k,w=e.slice(b,N),A.index-=b}else{h.lastIndex=0;var A=h.exec(w),S=1}if(A){f&&(m=A[1]?A[1].length:0);var P=A.index+m,A=A[0].slice(m),O=P+A.length,E=w.slice(0,P),_=w.slice(O),C=[k,S];E&&(++k,b+=E.length,C.push(E));var M=new t(u,d?r.tokenize(A,d):A,y,A,p);if(C.push(M),_&&C.push(_),Array.prototype.splice.apply(a,C),1!=S&&r.matchGrammar(e,a,n,k,b,!0,u),o)break}else if(o)break}}}}},tokenize:function(e,t){var a=[e],n=t.rest;if(n){for(var i in n)t[i]=n[i];delete t.rest}return r.matchGrammar(e,a,t,0,0,!1),a},hooks:{all:{},add:function(e,t){var a=r.hooks.all;a[e]=a[e]||[],a[e].push(t)},run:function(e,t){var a=r.hooks.all[e];if(a&&a.length)for(var n,i=0;n=a[i++];)n(t)}},Token:t};if(e.Prism=r,t.stringify=function(e,a,n){if("string"==typeof e)return e;if("Array"===r.util.type(e))return e.map(function(n){return t.stringify(n,a,e)}).join("");var i={type:e.type,content:t.stringify(e.content,a,n),tag:"span",classes:["token",e.type],attributes:{},language:a,parent:n};if(e.alias){var l="Array"===r.util.type(e.alias)?e.alias:[e.alias];Array.prototype.push.apply(i.classes,l)}r.hooks.run("wrap",i);var o=Object.keys(i.attributes).map(function(e){return e+'="'+(i.attributes[e]||"").replace(/"/g,"&quot;")+'"'}).join(" ");return"<"+i.tag+' class="'+i.classes.join(" ")+'"'+(o?" "+o:"")+">"+i.content+"</"+i.tag+">"},!e.document)return e.addEventListener?(r.disableWorkerMessageHandler||e.addEventListener("message",function(t){var a=JSON.parse(t.data),n=a.language,i=a.code,l=a.immediateClose;e.postMessage(r.highlight(i,r.languages[n],n)),l&&e.close()},!1),r):r;var i=document.currentScript||[].slice.call(document.getElementsByTagName("script")).pop();return i&&(r.filename=i.src,r.manual||i.hasAttribute("data-manual")||("loading"!==document.readyState?window.requestAnimationFrame?window.requestAnimationFrame(r.highlightAll):window.setTimeout(r.highlightAll,16):document.addEventListener("DOMContentLoaded",r.highlightAll))),r}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism);
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 82ff7bfae..9f0b9a1a4 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -6,7 +6,6 @@ import { ENV_VARS } from 'apify-shared/consts';
import { DEFAULT_USER_AGENT } from './constants';
import { getTypicalChromeExecutablePath, isAtHome } from './utils';
import { getApifyProxyUrl } from './actor';
-import { registerBrowserForLiveView } from './puppeteer_live_view_server';
/* global process, require */
@@ -65,17 +64,6 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
* @property ${String} [puppeteerModule]
* Require path to a module to be used instead of default 'puppeteer'. This enables usage
* of various Puppeteer wrappers such as 'puppeteer-extra'.
- * @property {Boolean} [liveView=false]
- * If set to `true`, a `PuppeteerLiveViewServer` will be started to enable
- * screenshot and html capturing of visited pages using `PuppeteerLiveViewBrowser`.
- * @property {Object} [liveViewOptions]
- * Settings for `PuppeteerLiveViewBrowser` started using `launchPuppeteer()`.
- * @property {String} [liveViewOptions.liveViewId]
- * Custom ID of a browser instance in live view.
- * @property {Number} [liveViewOptions.screenshotTimeoutMillis=3000]
- * Time in milliseconds before a screenshot capturing
- * will time out and the actor continues with execution.
- * Screenshot capturing pauses execution within the given page.
*/
/**
@@ -201,9 +189,11 @@ export const launchPuppeteer = async (options = {}) => {
checkParamOrThrow(options.proxyUrl, 'options.proxyUrl', 'Maybe String');
checkParamOrThrow(options.useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');
checkParamOrThrow(options.puppeteerModule, 'options.puppeteerModule', 'Maybe String');
- checkParamOrThrow(options.liveView, 'options.liveView', 'Maybe Boolean');
- checkParamOrThrow(options.liveViewOptions, 'options.liveViewOptions', 'Maybe Object');
if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');
+ if (options.liveView || options.liveViewOptions) {
+ log.deprecated('Live view is deprecated in Apify.launchPuppeteer() and launchPuppeteerOptions. '
+ + 'Use options.useLiveView in PuppeteerPool. For live view with Apify.launchPuppeteer(), use Apify.LiveViewServer.');
+ }
const puppeteer = getPuppeteerOrThrow(options.puppeteerModule);
const optsCopy = Object.assign({}, options);
@@ -213,8 +203,7 @@ export const launchPuppeteer = async (options = {}) => {
// TODO: It's not clear that this works, keep eye on https://bugs.chromium.org/p/chromium/issues/detail?id=723233
optsCopy.args.push('--enable-resource-load-scheduler=false');
if (optsCopy.headless == null) {
- // Forcing headless with liveView, otherwise screenshots redirect user to new browser window
- optsCopy.headless = optsCopy.liveView || (process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1');
+ optsCopy.headless = process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';
}
if (optsCopy.useChrome && (optsCopy.executablePath === undefined || optsCopy.executablePath === null)) {
optsCopy.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
@@ -241,19 +230,7 @@ export const launchPuppeteer = async (options = {}) => {
optsCopy.args.push(`--user-agent=${userAgent}`);
}
- let browserPromise;
- if (optsCopy.proxyUrl) {
- browserPromise = launchPuppeteerWithProxy(puppeteer, optsCopy);
- } else {
- log.info('Launching Puppeteer', _.omit(optsCopy, LAUNCH_PUPPETEER_LOG_OMIT_OPTS));
- browserPromise = puppeteer.launch(optsCopy);
- }
-
- // Start LiveView server if requested
- if (optsCopy.liveView) {
- const browser = await browserPromise;
- return registerBrowserForLiveView(browser, optsCopy.liveViewOptions).then(() => wrapped);
- }
-
- return wrapped;
+ if (optsCopy.proxyUrl) return launchPuppeteerWithProxy(puppeteer, optsCopy);
+ log.info('Launching Puppeteer', _.omit(optsCopy, LAUNCH_PUPPETEER_LOG_OMIT_OPTS));
+ return puppeteer.launch(optsCopy);
};
diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.js
deleted file mode 100644
index 9c7f622b9..000000000
--- a/src/puppeteer_live_view_client.js
+++ /dev/null
@@ -1,625 +0,0 @@
-import { checkParamOrThrow } from 'apify-client/build/utils';
-
-// Everything gets destroyed slowly to make the UX better
-const DESTROY_FADEOUT_MILLIS = 2000;
-
-/* global document, window */
-
-/**
- * Creates a page line in the Index.
- *
- * Used both client and server-side.
- *
- * @param {string} id
- * @param {string} url
- * @returns {string} html
- * @ignore
- */
-const createPage = (id, url) => `
- <tr id="${id}" class="page">
- <td class="status"></td>
- <td class="url">${url}</td>
- <td class="more"><i class="material-icons">search</i></td>
- </tr>
-`;
-
-/**
- * Turns a Map of Pages into an Array of page rows.
- * = All Pages of a single PuppeteerLiveViewBrowser.
- *
- * Used both client and server-side.
- *
- * @param {Map<String,Page>} pages
- * @returns {string} html
- * @ignore
- */
-const createPageCollection = (pages) => {
- const pageRows = [];
- pages.forEach((page, id) => {
- pageRows.push(createPage(id, page.url()));
- });
- return `
- <table class="page-collection">
- <thead>
- <th class="status">Status</th>
- <th class="url">URL</th>
- <th class="more"></th>
- </thead>
- <tbody>${pageRows.join('\n')}</tbody>
- </table>
- `;
-};
-
-/**
- * Creates a browser line in the Index.
- *
- * Used both client and server-side.
- *
- * @param {string} id Browser ID.
- * @param {Map<String,Page>} pages
- * @returns {string} html
- * @ignore
- */
-const createBrowser = (id, pages) => `<div class="browser" id="${id}"><h3>Browser: ${id}</h3>${createPageCollection(pages)}</div>`;
-
-/**
- * Turns a Set of Browsers into an Array of browser divs.
- * = All PuppeteerLiveViewBrowsers of a PuppeteerLiveViewServer.
- *
- * Used both client and server-side.
- *
- * @param {Set} browsers
- * @returns {string} html
- * @ignore
- */
-const createBrowserCollection = (browsers) => {
- const browserDivs = [];
- browsers.forEach(({ id, pages }) => {
- browserDivs.push(createBrowser(id, pages));
- });
- return `<div class="browser-collection">${browserDivs.join('\n')}</div>`;
-};
-
-/**
- * Generates an HTML body for the Index page.
- * A list of PuppeteerLiveViewBrowsers and their Pages.
- *
- * Used only server-side.
- *
- * @param {Set} browsers
- * @returns {string} html
- * @ignore
- */
-export const indexPage = (browsers) => {
- return createBrowserCollection(browsers);
-};
-
-/**
- * Returns a body of a Page consisting of a serialized
- * image and a HTML representation.
- *
- * Used only server-side.
- *
- * @param {Buffer} imageBuffer
- * @returns {string} html
- * @ignore
- */
-export const detailPage = ({ id, url, image, html }) => {
- const chars = { '<': '&lt', '>': '&gt', '&': '&amp' };
- const escapedHtml = html.replace(/[<>&]/g, m => chars[m]);
-
- return `
-<div id="${id}">
- <h2>Page detail</h2>
- <table class="page-detail">
- <tbody>
- <tr>
- <td>URL:</td>
- <td><a href="${url}" target="m_blank">${url}</a></td>
- </tr>
- </tbody>
- </table>
-
- <div id="tabmenu" class="tab-menu">
- <ul>
- <li><a class="tab-link active" href="#" data-target="screenshot">Screenshot</a></li>
- <li><a class="tab-link" href="#" data-target="code">HTML Code</a></li>
- </ul>
- </div>
-
- <div id="screenshot" class="tab-content">
- <img src="data:image/png;base64, ${image.toString('base64')}" alt="Page screenshot" />
- </div>
-
- <div id="code" class="tab-content hidden">
- <pre>
- <code class="original-html">
- ${escapedHtml}
- </code>
- </pre>
- </div>
-</div>
-`;
-};
-
-export const errorPage = ({ id, url, error }) => {
- return `
-<div id="${id}">
- <h3>There has been an error on page: ${url}</h3>
- <div class="error">
- <h4>${error.message}</h4>
- </div>
-</div>
-
-`;
-};
-
-/**
- * The wsHandler() function encapsulates the whole client-side
- * messaging and rendering logic. All commands that the client
- * is able to receive and understand are listed in COMMANDS.
- *
- * @param {WebSocket} socket
- * @ignore
- */
-const wsHandler = (socket) => {
- // Get common elements
- const index = document.getElementById('index');
- const pageDetail = document.getElementById('page-detail');
- const backLink = document.getElementById('back-link');
-
- // A client implementation of the server's sendCommand()
- const sendCommand = (command, data) => {
- // Send only if socket is open
- if (socket.readyState === 1) {
- const payload = JSON.stringify({ command, data });
- socket.send(payload);
- }
- };
-
- const COMMANDS = {
- // Renders the Index Page - a list of browsers and their pages
- renderIndex: ({ html }) => {
- index.innerHTML = html;
- index.insertAdjacentHTML('afterbegin', '<h2>Browser dashboard</h2>');
- index.querySelectorAll('.page').forEach((page) => {
- page.onclick = () => sendCommand('renderPage', {
- id: page.getAttribute('id'),
- });
- const status = page.querySelector('td.status');
- if (status) status.innerHTML = '<i class="material-icons rotating">cached</i><span>Running</span>';
- });
- },
- // Renders the Page Detail - where screenshots and HTML are shown
- renderPage: ({ html }) => {
- index.classList.add('hidden');
- backLink.classList.remove('hidden');
- pageDetail.classList.remove('hidden');
- pageDetail.innerHTML = html;
-
- const tabMenuItems = document.querySelectorAll('.tab-menu .tab-link');
-
- // Set onClick event for all menu tabs
- tabMenuItems.forEach((item) => {
- item.onclick = (event) => {
- event.preventDefault();
-
- // Get clicked element
- event = event || window.event;
- const target = event.target || event.srcElement;
- const text = target.textContent || text.innerText;
-
- const contentId = target.getAttribute('data-target');
-
- // Get all tab content blocks and hide them
- const tabContents = document.getElementsByClassName('tab-content');
- for (let j = 0; j < tabContents.length; j++) {
- tabContents[j].classList.add('hidden');
- }
-
- // Get all tab menu links and remove active class
- const tabLinks = document.getElementsByClassName('tab-link');
- for (let j = 0; j < tabLinks.length; j++) {
- tabLinks[j].classList.remove('active');
- }
-
- // Marked the clicked tab menu item as active
- target.classList.add('active');
-
- // Show the correct content block;
- document.getElementById(contentId).classList.remove('hidden');
- };
- });
- },
- // Adds a browser to the list
- createBrowser: ({ id }) => {
- const browsers = index.querySelector('.browser-collection');
- browsers.insertAdjacentHTML('afterbegin', createBrowser(id));
- },
- // Removes a browser from the list after fade-out
- destroyBrowser: ({ id }) => {
- const browser = document.getElementById(id);
- browser.classList.add('destroyed');
- // Do not remove immediately
- setTimeout(() => browser.remove(), DESTROY_FADEOUT_MILLIS);
- },
- // Adds a page to it's parent browser
- createPage: ({ id, browserId, url }) => {
- const pages = document.getElementById(browserId).querySelector('.page-collection tbody');
- pages.insertAdjacentHTML('afterbegin', createPage(id, url));
- const page = document.getElementById(id);
-
- // const button = page.querySelector('td.more i');
- page.onclick = () => sendCommand('renderPage', {
- id: page.getAttribute('id'),
- });
-
- const status = page.querySelector('td.status');
- if (status) status.innerHTML = '<i class="material-icons orange">watch_later</i><span class="orange">Starting</span>';
- },
- // Updates page URL on navigation
- updatePage: ({ id, url }) => {
- const page = document.getElementById(id);
-
- const spanUrl = page.querySelector('td.url');
- if (spanUrl) spanUrl.innerHTML = url;
-
- const status = page.querySelector('td.status');
- if (status) status.innerHTML = '<i class="material-icons rotating">cached</i><span>Running</span>';
- },
- // Removes a page from a browser after fade-out
- destroyPage: ({ id }) => {
- const page = document.getElementById(id);
-
- const status = page.querySelector('td.status');
- if (status) status.innerHTML = '<i class="material-icons">check_circle</i><span>Finished</span>';
-
- const button = page.querySelector('td.more i');
- button.remove();
-
- page.classList.add('destroyed');
- // Do not remove immediately since it can happen pretty fast
- // and the page only pops in the list for a split second.
- setTimeout(() => page.remove(), DESTROY_FADEOUT_MILLIS);
- },
- // Handles errors
- error: ({ message, status }) => {
- console.error(`${status}: ${message}`); // eslint-disable-line
- },
- };
-
- socket.onmessage = (e) => {
- let message;
- try {
- message = JSON.parse(e.data);
- } catch (err) {
- return console.error('Unable to parse message from server:', e.data); // eslint-disable-line
- }
-
- // Validate and invoke requested command
- const { command, data } = message;
- if (!command) return console.error('Invalid command:', command); // eslint-disable-line
- const fn = COMMANDS[command];
- if (!fn || typeof fn !== 'function') return console.error('Command not recognized by client:', command); // eslint-disable-line
- fn(data);
- };
-
- socket.onclose = () => {
- index.insertAdjacentHTML('beforeend', '<div class="message">Actor finished</div>');
- const statusElements = document.querySelectorAll('td.status');
- statusElements.forEach((e) => {
- e.innerHTML = '<i class="material-icons">check_circle</i><span>Finished</span>';
- });
- };
-
- socket.onerror = (err) => {
- console.error(err); //eslint-disable-line
- };
-
- // The Index Page and Page Detail are not rerendered every time since
- // it complicates WebSocket communication (adding / removing listeners).
- // Instead, clicking on a page in the Index simply hides the Index
- // and shows Page Detail and the Back link.
- // Clicking the back link hides the detail and shows the Index Page.
- backLink.onclick = () => {
- pageDetail.classList.add('hidden');
- backLink.classList.add('hidden');
- index.classList.remove('hidden');
- const id = pageDetail.firstElementChild.getAttribute('id');
- sendCommand('quitPage', { id });
- };
-};
-
-/**
- * Template for a basic layout of a HTML page that enables WebSocket
- * communication. Constants and JavaScript are templated directly into
- * the HTML. This HTML page is the only piece of data sent over HTTP.
- * All other communication takes place over WebSockets.
- *
- * @param {String} url Url of the WebSocket server
- * @returns {string} html
- * @ignore
- */
-export const layout = (url) => {
- checkParamOrThrow(url, 'url', 'String');
-
- return `
-<!doctype html>
-<html>
-<head>
- <meta charset="utf-8">
- <title>Puppeteer Live View Server</title>
- <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
- <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
- <style>
- .page {
- background: #fff;
- transition: background 0.3s ease-in;
- }
- .page:hover {
- cursor: pointer;
- background: #f0f0f2;
- }
- .destroyed {
- color: #660000;
- visibility: hidden;
- opacity: 0;
- transition: visibility 0s ${DESTROY_FADEOUT_MILLIS / 1000}s, opacity ${DESTROY_FADEOUT_MILLIS / 1000}s linear;
- }
- .hidden {
- display: none;
- }
- .original-html {
- white-space: pre-wrap;
- word-wrap: break-word;
- }
- body {
- margin: 0;
- padding: 0;
- font-family: Graphik, sans-serif;
- }
- header {
- width: 100%;
- padding: 5px 15px;
- background: #1a346d;
- color: white;
- }
- h1 {
- margin: 10px 0;
- font-size: 28px;
- font-weight: 400;
- }
- h2 {
- font-size: 22px;
- font-weight: 500;
- margin-bottom: 25px;
- }
- h3 {
- font-size: 15px;
- font-weight: 500;
- }
- main {
- padding: 5px 15px;
- }
- .browser {
- margin-bottom: 40px;
- }
- a {
- color: #00A6D0;
- text-decoration: none;
- transition: all .2s ease-in-out;
- }
- a:focus, a:hover {
- color: #006984;
- text-decoration: underline;
- }
- #back-link {
- margin: 10px 0;
- cursor: pointer;
- width: fit-content;
- }
- #back-link i {
- background: #1a346d;
- color: white;
- border-radius: 100%;
- border: 1px solid #1a346d;
- font-size: 14px;
- padding: 3px;
- float: left;
- margin-top: -2px;
- margin-right: 5px;
- transition: all .3s ease-in;
- }
- #back-link:hover i {
- background: white;
- border-color: #1a346d;
- color: #1a346d;
- }
- table {
- border-collapse: collapse;
- }
- table, th, td {
- border: 1px solid #DDD;
- }
- thead {
- background: #F8F9F9;
- }
- th,td {
- padding: 8px;
- font-size: 14px;
- vertical-align: middle;
- height: 24px;
- }
- th {
- font-weight: 500;
- color: #9FA5A9;
- }
- th.status {
- min-width: 100px;
- }
- th.url {
- min-width: 400px;
- }
- th.more {
- min-width: 42px;
- }
- td.status {
- color: #69C242;
- min-width: 100px;
- }
- td.status i {
- float: left;
- }
- td.status span {
- margin-left: 10px;
- line-height: 24px;
- }
- td.url {
- min-width: 400px;
- }
- td.more {
- color: #00a6d0;
- text-align: center;
- position: relative;
- }
- td.more:focus, td.more:hover {
- color: #006984;
- text-decoration: underline;
- }
- td.more i {
- cursor: pointer;
- position: absolute;
- left: 18px;
- top: 7px;
- }
- table.page-detail {
- border: none;
- }
- table.page-detail td {
- border: none;
- padding-left: 0;
- padding-right: 20px;
- font-size: 14px;
- }
- table.page-detail td:first-child {
- font-weight: 700
- }
- .tab-menu {
- margin: 30px 0 0;
- }
- .tab-menu ul {
- list-style-type: none;
- border-bottom: 1px solid #ddd;
- margin-bottom: 0;
- padding-left: 0;
- text-transform: uppercase;
- }
- .tab-menu ul:after, .tab-menu ul:before {
- content: " ";
- display: table;
- }
- .tab-menu ul:after {
- clear: both;
- }
- .tab-menu li {
- float: left;
- margin-right: 10px;
- display: block;
- margin-bottom: -1px;
- }
- .tab-link {
- padding: 10px 15px;
- display: block;
- color: #9fa5a9;
- text-decoration: none;
- border: 1px solid #ddd;
- transition: background .3s ease-in;
- border-radius: 3px 3px 0 0;
- font-size: 13px;
- font-weight: 700;
- line-height: 1.428571429;
- }
- .tab-link:focus, .tab-link:hover {
- color: #9fa5a9;
- border-color: #ddd;
- text-decoration: none;
- background-color: #eee;
- }
- .tab-link.active {
- border-bottom-color: #fff;
- color: #11181c;
- background: #fff;
- }
- .tab-content {
- padding: 20px;
- border-top: 0;
- border: 1px solid #ddd;
- border-top: none;
- visibility: visible;
- opacity: 1;
- transition: opacity 0.3s ease-in;
- }
- .tab-content.hidden {
- display: block;
- visibility: hidden;
- opacity: 0;
- height: 0;
- border: none;
- padding: 0;
- }
- .tab-content.hidden pre {
- display: none;
- }
- .tab-content img{
- max-width: 100%;
- }
- .message {
- color: #69C242;
- }
- .orange {
- color: #f0ad4e;
- }
- @-webkit-keyframes rotating {
- from { -webkit-transform: rotate(0deg); }
- to { -webkit-transform: rotate(-360deg); }
- }
- @-moz-keyframes rotating {
- from { -moz-transform: rotate(0deg); }
- to { -moz-transform: rotate(-360deg); }
- }
- @keyframes rotating {
- from {transform:rotate(0deg);}
- to {transform:rotate(-360deg);}
- }
-
- .rotating {
- -webkit-animation: rotating 2s linear infinite;
- -moz-animation: rotating 2s linear infinite;
- animation: rotating 2s linear infinite;
- }
- </style>
-</head>
-<body>
- <header>
- <h1>Puppeteer Live View</h1>
- </header>
- <main>
- <div id="back-link" class="hidden">
- <i class="material-icons">arrow_back</i>
- <span>dashboard</span>
- </div>
- <div id="index">Waiting for WebSocket connection.</div>
- <div id="page-detail" class="hidden"></div>
- </main>
- <script>
- const ws = new WebSocket("${url.replace('https', 'wss').replace('http', 'ws')}");
- const DESTROY_FADEOUT_MILLIS = ${DESTROY_FADEOUT_MILLIS};
- const createPage = ${createPage.toString()};
- const createPageCollection = ${createPageCollection.toString()};
- const createBrowser = ${createBrowser.toString()};
- (${wsHandler.toString()})(ws);
- </script>
-</body>
-</html>
-`;
-};
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
deleted file mode 100644
index c2ea52cd6..000000000
--- a/src/puppeteer_live_view_server.js
+++ /dev/null
@@ -1,479 +0,0 @@
-import http from 'http';
-import EventEmitter from 'events';
-import Promise from 'bluebird';
-import WebSocket from 'ws';
-import log from 'apify-shared/log';
-import { promisifyServerListen } from 'apify-shared/utilities';
-import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
-import { checkParamOrThrow } from 'apify-client/build/utils';
-import { layout, indexPage, detailPage, errorPage } from './puppeteer_live_view_client';
-
-const DEFAULT_SCREENSHOT_TIMEOUT_MILLIS = 3000;
-
-/**
- * LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides
- * an API to capture screenshots and HTML of the Browser's Pages. It uses
- * EventEmitter API to communicate relevant Page events.
- *
- * @param {Browser} browser A Puppeteer Browser instance.
- * @param {String} [options.id] A unique ID of the LiveViewBrowser.
- * @param {Number} [options.screenshotTimeoutMillis] Max time allowed for the screenshot taking process.
- * @ignore
- */
-export class PuppeteerLiveViewBrowser extends EventEmitter {
- constructor(browser, opts = {}) {
- super();
- this.browser = browser;
- this.pages = new Map(); // To track all pages and their creation order for listing
- this.pageIds = new WeakMap(); // To avoid iteration over pages
- this.loadedPages = new WeakSet(); // To just track loaded state
- this.pageIdCounter = 0;
-
- checkParamOrThrow(opts.id, 'opts.id', 'String');
- checkParamOrThrow(opts.screenshotTimeoutMillis, 'opts.screenshotTimeoutMillis', 'Maybe Number');
- this.id = opts.id;
- this.screenshotTimeoutMillis = opts.screenshotTimeoutMillis || DEFAULT_SCREENSHOT_TIMEOUT_MILLIS;
-
- // Since the page can be in any state when the user requests
- // a screenshot, we need to keep track of it ourselves.
- browser.on('targetcreated', this._onTargetCreated.bind(this));
- // Clean up resources after page close
- browser.on('targetdestroyed', this._onTargetDestroyed.bind(this));
- // Clean up resources after browser close (in Server)
- browser.on('disconnected', this._onBrowserDisconnected.bind(this));
- }
-
- /**
- * Initiates screenshot and HTML capturing of a Page by attaching
- * a listener to Page "load" events. If the first Page is already loaded,
- * it captures immediately.
- *
- * This function is invoked as a response to a Client side command "renderPage",
- * which is issued by clicking on a Page listing on the Browser Index page.
- * Once capturing starts, new screenshot and HTML will be served to Client
- * with each Page load.
- * @param {Page} page Puppeteer Page instance.
- */
- startCapturing(page) {
- const id = this.pageIds.get(page);
- if (!id) return;
- const capture = () => {
- log.debug(`Capturing page. ID: ${id}`);
- this._getScreenshotAndHtml(page)
- .then(({ image, html }) => {
- this.emit(id, {
- id,
- url: page.url(),
- image,
- html,
- });
- })
- .catch((err) => {
- this.emit(id, {
- id,
- url: page.url(),
- error: err,
- });
- log.error(err);
- });
- };
- // Capture immediately for loaded pages
- if (this.loadedPages.has(page)) capture();
- // Setup recurrent capturing
- page.on('load', () => {
- if (this.listenerCount(id)) capture();
- });
- }
-
- /**
- * Clears the capturing listener setup by startCapturing() to prevent
- * unnecessary load on both Server and Client.
- *
- * Client side, the function is invoked by clicking the "back" button
- * on a page detail (screenshot + HTML).
- *
- * @param {Page} page
- */
- stopCapturing(page) {
- const id = this.pageIds.get(page);
- this.removeAllListeners(id);
- }
-
- /**
- * The getScreenshotAndHtml method simply retrieves the page's HTML
- * content, takes a screenshot and returns both as a promise.
- * Unfortunately, nothing prevents the Page from being closed while
- * the screenshot is being taken, which results into error.
- * Therefore, the method prevents the page from being closed
- * by replacing its close method and handling the page close
- * itself once the screenshot has been taken.
- *
- * @param {Page} page Puppeteer's Page
- * @returns {Promise<Buffer>} screenshot
- * @private
- * @ignore
- */
- _getScreenshotAndHtml(page) {
- // Replace page's close function to prevent a close
- // while the screenshot is being taken.
- const { close } = page;
- let closed;
- let closeArgs;
- let closeResolve;
- page.close = (...args) => {
- if (!closed) closeArgs = args;
- closed = true;
- return new Promise((resolve) => {
- closeResolve = resolve;
- });
- };
-
- const cleanup = () => {
- // Replace the stolen close() method or call it,
- // if it should've been called externally.
- if (closed) {
- close.apply(page, closeArgs)
- .then(closeResolve);
- } else {
- page.close = close.bind(page);
- }
- };
-
- return Promise.props({
- image: page.screenshot(),
- html: page.content(),
- })
- .timeout(this.screenshotTimeoutMillis, 'Puppeteer Live View: Screenshot timed out.')
- .finally(cleanup);
- }
-
- /**
- * Handler invoked whenever a Target is created in a Puppeteer's Browser.
- * @param {Target} target
- * @private
- */
- _onTargetCreated(target) {
- if (target.type() !== 'page') return;
- target.page()
- .then((page) => {
- const id = `${this.id}_PAGE_${++this.pageIdCounter}`;
- this.pages.set(id, page);
- this.pageIds.set(page, id);
- this.emit('pagecreated', {
- id,
- browserId: this.id,
- url: page.url(),
- });
- page.on('load', () => {
- this.loadedPages.add(page); // Page is loaded
- });
- // Using "framenavigated" on main Frame since "targetchanged" does not seem reliable
- page.on('framenavigated', (frame) => {
- if (frame !== page.mainFrame()) return;
- this.loadedPages.delete(page);
- this.emit('pagenavigated', {
- id,
- url: frame.url(),
- });
- });
- })
- .catch(err => log.error(err));
- }
-
- /**
- * Handler invoked whenever a Target is destroyed in a Puppeteer's Browser.
- * @param {Target} target
- * @private
- */
- _onTargetDestroyed(target) {
- if (target.type() !== 'page') return;
- target.page()
- .then((page) => {
- const id = this.pageIds.get(page);
- this.pages.delete(id);
- this.emit('pagedestroyed', {
- id,
- });
- })
- .catch(err => log.error(err));
- }
-
- /**
- * Handler invoked whenever a Browser is destroyed / disconnected in Puppeteer.
- * @param {Browser} browser
- * @private
- */
- _onBrowserDisconnected(browser) {
- this.emit('disconnected', browser);
- }
-}
-
-let liveViewServer;
-/**
- * The registerBrowserForLiveView method should cover most use cases of starting a PuppeteerLiveViewServer.
- * It creates a single instance of the server on its first invocation and subsequent
- * invocations only add more browsers to the current server instance. Individual browsers
- * are assigned unique IDs that will be used in displaying the browsers in an HTML index
- * available at the server's root route.
- *
- * The ID is customizable to better identify individual browsers.
- *
- * The promise will resolve when the server starts listening or when the Puppeteer Browser
- * becomes available. Whichever comes later.
- *
- * @param {Promise<Browser>} browserPromise A Promise for a Puppeteer's Browser.
- * @param {Object} [opts] Options to pass down to PuppeteerLiveViewBrowser constructor.
- * @param {String} [opts.id] Custom ID to be used with the browser instance.
- * @param {String} [opts.screenshotTimeoutMillis] Max time allowed for the screenshot taking process.
- * @returns {Promise<PuppeteerLiveViewServer>}
- * @ignore
- */
-export const registerBrowserForLiveView = (browserPromise, opts = {}) => {
- let serverPromise = Promise.resolve();
- if (!liveViewServer) {
- liveViewServer = new PuppeteerLiveViewServer();
- serverPromise = liveViewServer.startServer();
- }
- const optsCopy = Object.assign({}, opts);
- if (!optsCopy.id) optsCopy.id = `BROWSER_${++liveViewServer.browserIdCounter}`;
-
- browserPromise
- .then((browser) => {
- const lvb = new PuppeteerLiveViewBrowser(browser, optsCopy);
- liveViewServer.addBrowser(lvb);
- return liveViewServer;
- });
-
- return Promise.all([serverPromise, browserPromise]).then(() => liveViewServer); // Return PuppeteerLiveViewServer instance
-};
-
-/**
- * sendCommand() encapsulates simple WebSocket communication logic.
- *
- * @example
- *
- * A command is sent as JSON:
- *
- * {
- * "command": "renderPage",
- * "data": {
- * "param1": "value1".
- * "param2": "value2"
- * }
- * }
- *
- * @param {WebSocket} socket
- * @param {String} command Name of requested command.
- * @param {Object} data Data to be sent.
- * @ignore
- */
-const sendCommand = (socket, command, data) => {
- const payload = JSON.stringify({ command, data });
- socket.send(payload, (err) => {
- if (err) log.error(err);
- });
-};
-
-/**
- * Enables live monitoring of actor execution by spawning a web server that responds with a list
- * of available browsers at its root path. Once the user chooses a browser, PuppeteerLiveViewServer will
- * periodically serve screenshots of the selected browser's latest loaded page.
- *
- * The server will be started on a port defined by the `CONTAINER_PORT` environment variable,
- * or a randomly-selected port.
- * @ignore
- */
-export default class PuppeteerLiveViewServer extends EventEmitter {
- constructor() {
- super();
-
- const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];
- this.liveViewPort = parseInt(containerPort, 10);
- if (!(this.liveViewPort >= 0 && this.liveViewPort <= 65535)) {
- throw new Error(`Cannot start LiveViewServer - invalid port specified by the ${
- ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);
- }
-
- this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];
-
- this.browsers = new Set();
- this.browserIdCounter = 0;
- this.httpServer = null;
- }
-
- /**
- * Adds an instance of PuppeteerLiveViewBrowser (not Puppeteer.Browser)
- * to the server's list of managed browsers.
- *
- * @param {PuppeteerLiveViewBrowser} browser
- */
- addBrowser(browser) {
- this.browsers.add(browser);
- this.emit('browsercreated', browser);
- browser.on('disconnected', () => {
- this.deleteBrowser(browser);
- if (!this.browsers.size) this.httpServer.close();
- });
- }
-
- /**
- * Removes an instance of PuppeteerLiveViewBrowser (not Puppeteer.Browser)
- * from the server's list of managed browsers.
- * @param {PuppeteerLiveViewBrowser} browser
- */
- deleteBrowser(browser) {
- this.browsers.delete(browser);
- this.emit('browserdestroyed', browser);
- browser.removeAllListeners();
- }
-
- /**
- * Starts an HTTP and a WebSocket server on a port defined in `CONTAINER_PORT` environment
- * variable or a randomly-selected port.
- *
- * @return {Promise} resolves when HTTP server starts listening
- */
- startServer() {
- const server = http.createServer(this._httpRequestListener.bind(this));
- const wss = new WebSocket.Server({ server });
-
- wss.on('connection', this._wsRequestListener.bind(this));
-
- return promisifyServerListen(server)(this.liveViewPort)
- .then(() => {
- log.info(`Live view is now available at ${this.liveViewUrl}`);
- this.httpServer = server;
- });
- }
-
- /**
- * Request handler function that returns a simple HTML index page
- * with embedded JavaScript that will establish a WebSocket connection
- * between the server and the client.
- * @param {http.IncomingMessage} req
- * @param {http.ServerResponse} res
- * @private
- * @ignore
- */
- _httpRequestListener(req, res) {
- const body = layout(this.liveViewUrl);
- res.writeHead(200, {
- 'Content-Type': 'text/html',
- 'Content-Length': Buffer.byteLength(body),
- });
- res.end(body);
- }
-
- /**
- * This function fires with each new WebSocket connection and manages
- * the socket's messages.
- * @param {WebSocket} ws
- * @private
- * @ignore
- */
- _wsRequestListener(ws) {
- const BAD_REQUEST = {
- message: 'Bad Request',
- status: 400,
- };
- const NOT_FOUND = {
- message: 'Not Found',
- status: 404,
- };
-
- // Traverses browsers to find a page with ID
- const findPage = (id) => {
- let page;
- let browser;
- for (const b of this.browsers) { // eslint-disable-line
- if (b.pages.has(id)) {
- page = b.pages.get(id);
- browser = b;
- break;
- }
- }
- return [browser, page];
- };
-
- const COMMANDS = {
- renderIndex: () => sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) }),
- // When the user selects a Page from the Index
- renderPage: (msg) => {
- const { id } = msg.data || {};
- const [browser, page] = findPage(id);
- if (!page) return sendCommand(ws, 'error', NOT_FOUND);
- browser.startCapturing(page);
- browser.on(id, (pageData) => {
- if (pageData.error) sendCommand(ws, 'renderPage', { html: errorPage(pageData) });
- else sendCommand(ws, 'renderPage', { html: detailPage(pageData) });
- });
- },
- // When the user clicks on the back button
- quitPage: (msg) => {
- const { id } = msg.data || {};
- const [browser, page] = findPage(id);
- if (!page) return; // no need to send error
- browser.stopCapturing(page);
- },
- };
-
- log.debug('WebSocket connection to Puppeteer Live View established.');
- ws.on('message', (msg) => {
- try {
- msg = JSON.parse(msg);
- } catch (err) {
- return sendCommand(ws, 'error', BAD_REQUEST);
- }
- // Validate command and send response
- const { command } = msg;
- const fn = COMMANDS[command];
- if (!fn || typeof fn !== 'function') return sendCommand(ws, 'error', BAD_REQUEST);
- fn(msg);
- });
- this._setupCommandHandles(ws);
- // Send first Index after WS connection is established
- sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });
- }
-
- /**
- * Invoked from the _wsRequestListener, this function only groups
- * all browser related event handling into a single package.
- * @param {WebSocket} ws
- * @private
- * @ignore
- */
- _setupCommandHandles(ws) {
- const attachListeners = (browser) => {
- const createListener = p => sendCommand(ws, 'createPage', p);
- const destroyListener = p => sendCommand(ws, 'destroyPage', p);
- const updateListener = p => sendCommand(ws, 'updatePage', p);
- browser.on('pagecreated', createListener);
- browser.on('pagedestroyed', destroyListener);
- browser.on('pagenavigated', updateListener);
- // Clean up to prevent listeners firing into closed sockets (on page refresh)
- ws.on('close', () => {
- browser.removeListener('pagecreated', createListener);
- browser.removeListener('pagedestroyed', destroyListener);
- browser.removeListener('pagenavigated', updateListener);
- });
- };
- // Since this function fires when a socket opens, some browsers
- // and pages already exist and we need to get messages about those too.
- this.browsers.forEach(attachListeners);
-
- const createListener = (browser) => {
- attachListeners(browser);
- sendCommand(ws, 'createBrowser', { id: browser.id });
- };
- const destroyListener = (browser) => {
- sendCommand(ws, 'destroyBrowser', { id: browser.id });
- };
- this.on('browsercreated', createListener);
- this.on('browserdestroyed', destroyListener);
- // Clean up to prevent listeners firing into closed sockets (on page refresh)
- ws.on('close', () => {
- this.removeListener('browsercreated', createListener);
- this.removeListener('browserdestroyed', destroyListener);
- });
- }
-}
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index f3f81dcba..8a491015d 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -31,7 +31,7 @@ const DEFAULT_OPTIONS = {
launchPuppeteerFunction: launchPuppeteerOptions => launchPuppeteer(launchPuppeteerOptions),
recycleDiskCache: false,
- liveView: false,
+ useLiveView: false,
};
const mkdtempAsync = util.promisify(fs.mkdtemp);
@@ -466,6 +466,7 @@ class PuppeteerPool {
* @return {Promise<Page>}
*/
async newPage() {
+ if (this.liveViewServer && !this.liveViewServer.isRunning) await this.liveViewServer.start();
let idlePage;
// We don't need to check whether options.reusePages is true,
// because if it's false, the array will be empty and the loop will never start.
@@ -596,7 +597,10 @@ class PuppeteerPool {
}
await Promise.all(dirDeletionPromises);
} catch (err) {
- log.exception(err, 'PuppeteerPool: Cannot close the browsers');
+ log.exception(err, 'PuppeteerPool: Cannot close the browsers.');
+ }
+ if (this.liveViewServer) {
+ await this.liveViewServer.stop().catch(err => log.exception(err, 'PuppeteerPool: Cannot close LiveViewServer.'));
}
}
@@ -670,11 +674,10 @@ class PuppeteerPool {
}
async _serveLiveView(page) {
- if (!this.liveViewServer.isRunning) await this.liveViewServer.start();
const browser = page.browser();
- const instance = this._findInstanceByBrowser(browser);
+ const instance = await this._findInstanceByBrowser(browser);
// Only take snapshots in the most recently opened browser.
- if (instance.id !== this.browserCounter) return;
+ if (instance.id !== this.browserCounter - 1) return;
await this.liveViewServer.serve(page);
}
}
diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.js
index d9aad1e45..74c1a807d 100644
--- a/test/live_view/live_view_server.js
+++ b/test/live_view/live_view_server.js
@@ -3,7 +3,7 @@ import { promisify } from 'util';
import rqst from 'request-promise-native';
import fs from 'fs-extra';
import { expect } from 'chai';
-import io from 'socket.io-client'; // eslint-disable-line
+import io from 'socket.io-client';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import Apify from '../../build/index';
import { LOCAL_STORAGE_DIR } from '../_helper';
@@ -54,16 +54,11 @@ describe('LiveViewServer', () => {
it('should connect over websocket', async () => {
const socket = io(BASE_URL);
- const connected = new Promise((resolve) => {
- socket.on('connect', async () => {
- expect(lvs.hasClients()).to.be.eql(true);
- await lvs.stop();
- socket.close();
- resolve();
- });
- });
await lvs.start();
- await connected;
+ await new Promise(resolve => socket.on('connect', resolve));
+ expect(lvs.hasClients()).to.be.eql(true);
+ socket.close();
+ await lvs.stop();
});
describe('when connected', () => {
@@ -107,7 +102,7 @@ describe('LiveViewServer', () => {
it('should not store more than maxScreenshotFiles screenshots', async () => {
const snapshots = [];
socket.on('snapshot', s => snapshots.push(s));
- await Promise.all(Array(3).fill(null).map(_ => lvs.serve(fakePage)));
+ await Promise.all(Array(3).fill(null).map(() => lvs.serve(fakePage)));
const screenshots = await Promise.all(snapshots
.sort((a, b) => a.screenshotIndex - b.screenshotIndex)
.map((snap, idx) => {
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 615861525..696920e8e 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -1,4 +1,5 @@
-import { expect } from 'chai';
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
import proxy from 'proxy';
import http from 'http';
import util from 'util';
@@ -11,6 +12,8 @@ import Apify from '../build/index';
import * as actor from '../build/actor';
import * as utils from '../build/utils';
+chai.use(chaiAsPromised);
+
let prevEnvHeadless;
let proxyServer;
let proxyPort; // eslint-disable-line no-unused-vars
@@ -65,20 +68,20 @@ after(function () {
describe('Apify.launchPuppeteer()', () => {
it('throws on invalid args', () => {
- expect(() => Apify.launchPuppeteer('some non-object')).to.throw(Error);
- expect(() => Apify.launchPuppeteer(1234)).to.throw(Error);
-
- expect(() => Apify.launchPuppeteer({ proxyUrl: 234 })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: {} })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'https://somehost:1234' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'xxx', useApifyProxy: true })).to.throw(Error);
-
- expect(() => Apify.launchPuppeteer({ args: 'wrong args' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ args: [12, 34] })).to.throw(Error);
+ expect(Apify.launchPuppeteer('some non-object')).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer(1234)).to.be.rejectedWith(Error);
+
+ expect(Apify.launchPuppeteer({ proxyUrl: 234 })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: {} })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: 'https://somehost:1234' })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ proxyUrl: 'xxx', useApifyProxy: true })).to.be.rejectedWith(Error);
+
+ expect(Apify.launchPuppeteer({ args: 'wrong args' })).to.be.rejectedWith(Error);
+ expect(Apify.launchPuppeteer({ args: [12, 34] })).to.be.rejectedWith(Error);
});
it('opens https://www.example.com', () => {
@@ -231,6 +234,6 @@ describe('Apify.launchPuppeteer()', () => {
headless: true,
};
- expect(() => Apify.launchPuppeteer(opts)).to.throw(Error);
+ expect(Apify.launchPuppeteer(opts)).to.be.rejectedWith(Error);
});
});
diff --git a/test/puppeteer_live_view_server.js b/test/puppeteer_live_view_server.js
deleted file mode 100644
index af9c3ca08..000000000
--- a/test/puppeteer_live_view_server.js
+++ /dev/null
@@ -1,116 +0,0 @@
-import http from 'http';
-import { expect, assert } from 'chai';
-import puppeteer from 'puppeteer';
-import { ENV_VARS } from 'apify-shared/consts';
-import Apify from '../build/index';
-import PuppeteerLiveViewServer, { PuppeteerLiveViewBrowser } from '../build/puppeteer_live_view_server';
-
-const PORT = 0;
-const URL = 'thisshouldfailonclientonly';
-
-const httpGet = (port) => {
- return new Promise((resolve, reject) => {
- const opts = {
- host: 'localhost',
- port,
- };
- http.get(opts, (res) => {
- let body = '';
- res.on('data', (d) => {
- body += d;
- });
- res.on('end', () => resolve(body));
- res.on('error', reject);
- });
- });
-};
-
-beforeEach(() => {
- process.env[ENV_VARS.CONTAINER_PORT] = PORT;
- process.env[ENV_VARS.CONTAINER_URL] = URL;
-});
-
-describe('Starting the PuppeteerLiveViewServer', () => {
- it('should start using Apify.launchPuppeteer()', async () => {
- const customPort = 1234;
-
- process.env[ENV_VARS.CONTAINER_PORT] = customPort;
-
- return Apify.launchPuppeteer({ liveView: true })
- .then((browser) => {
- return httpGet(customPort)
- .catch(err => assert.fail(err, 'Server response.'))
- .then((body) => {
- expect(body.trim().substr(0, 15)).to.equal('<!doctype html>');
- })
- .then(() => browser.close());
- });
- });
- it('should start using PuppeteerLiveViewServer.startServer()', () => {
- const server = new PuppeteerLiveViewServer();
- return server.startServer()
- .then(() => {
- const { port } = server.httpServer.address();
- return httpGet(port);
- })
- .catch(err => assert.fail(err, 'Server response.'))
- .then((body) => {
- expect(body.trim().substr(0, 15)).to.equal('<!doctype html>');
- server.httpServer.close();
- });
- });
-});
-
-describe('Manipulate the PuppeteerLiveViewServer', () => {
- let server;
-
- beforeEach(() => {
- server = new PuppeteerLiveViewServer();
- return server.startServer();
- });
-
- afterEach(() => {
- server.httpServer.close();
- server = null;
- });
-
- it('should add and remove browsers', () => {
- let createHandler = 0;
- server.on('browsercreated', () => {
- createHandler++;
- });
-
- let destroyHandler = 0;
- server.on('browserdestroyed', () => {
- destroyHandler++;
- });
-
- const browsers = [];
-
- return puppeteer.launch()
- .then((browser) => {
- browsers.push(browser);
- return server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B1' }));
- })
- .then(() => {
- expect(server.browsers.size).to.equal(1);
- expect(createHandler).to.equal(1);
- })
- .then(() => puppeteer.launch())
- .then((browser) => {
- browsers.push(browser);
- return server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B2' }));
- })
- .then(() => {
- expect(server.browsers.size).to.equal(2);
- expect(createHandler).to.equal(2);
- })
- .then(() => server.deleteBrowser(server.browsers.values().next().value))
- .then(() => {
- expect(server.browsers.size).to.equal(1);
- expect(createHandler).to.equal(2);
- expect(destroyHandler).to.equal(1);
- })
- .then(() => Promise.all(browsers.map(browser => browser.close())));
- });
-});
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index 2e011a74f..51bc58159 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -575,4 +575,40 @@ describe('PuppeteerPool', () => {
}
});
});
+
+ describe('enables LiveView', () => {
+ before(() => {
+ process.env[ENV_VARS.IS_AT_HOME] = '1';
+ });
+ after(() => {
+ delete process.env[ENV_VARS.IS_AT_HOME];
+ });
+ it('should work', async () => {
+ const serveCalledWith = [];
+ let started = 0;
+ let stopped = 0;
+ const pool = new Apify.PuppeteerPool({
+ launchPuppeteerOptions: { headless: true },
+ useLiveView: true,
+ });
+ pool.liveViewServer.start = async () => {
+ started++;
+ pool.liveViewServer._isRunning = true; // eslint-disable-line no-underscore-dangle
+ };
+ pool.liveViewServer.serve = async arg => serveCalledWith.push(arg);
+ pool.liveViewServer.stop = async () => stopped++;
+
+ for (let i = 0; i < 3; i++) {
+ const page = await pool.newPage();
+ await pool.recyclePage(page);
+ }
+
+ expect(started).to.be.eql(1);
+ expect(serveCalledWith).to.have.lengthOf(3);
+ serveCalledWith.forEach(item => expect(item.constructor.name === 'Page'));
+
+ await pool.destroy();
+ expect(stopped).to.be.eql(1);
+ });
+ });
});