Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactor code based on commit comments
Ondra Urban0f918da897bf
diff --git a/examples/synchronous_run.js b/examples/synchronous_run.jsindex dcb3d3ab2..731e703df 100644--- a/examples/synchronous_run.js+++ b/examples/synchronous_run.js@@ -25,7 +25,7 @@ Apify.main(async () => {const imageElementHandle = await iframe.$('.VideoColm img');// Get a screenshot of that image.- const imageBuffer = await imageElementHandle.getScreenshotAndHtml();+ const imageBuffer = await imageElementHandle.screenshot();// Save it as an output.await Apify.setValue('OUTPUT', imageBuffer, { contentType: 'image/jpeg' });diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 69bfc78b8..5f73c7dab 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -5,7 +5,7 @@ import log from 'apify-shared/log';import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';import { newPromise, getTypicalChromeExecutablePath } from './utils';import { getApifyProxyUrl } from './actor';-import { startPuppeteerLiveView } from './puppeteer_live_view_server';+import { registerBrowserForLiveView } from './puppeteer_live_view_server';/* global process, require */@@ -164,11 +164,9 @@ export const launchPuppeteer = (opts = {}) => {optsCopy.args = optsCopy.args || [];optsCopy.args.push('--no-sandbox');- if (optsCopy.headless === undefined || optsCopy.headless === null) {- // forcing headless with liveView, otherwise screenshots open a new browser window- optsCopy.headless = optsCopy.liveView === true- ? true- : process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';+ if (optsCopy.headless == null) {+ // forcing headless with liveView, otherwise screenshots redirect user to new browser window+ optsCopy.headless = optsCopy.liveView || (process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1');}if (optsCopy.useChrome && (optsCopy.executablePath === undefined || optsCopy.executablePath === null)) {optsCopy.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();@@ -201,7 +199,7 @@ export const launchPuppeteer = (opts = {}) => {const wrapped = newPromise().then(() => browserPromise);// start LiveView server if requested- if (optsCopy.liveView) return startPuppeteerLiveView(wrapped).then(() => wrapped);+ if (optsCopy.liveView) return registerBrowserForLiveView(wrapped, optsCopy.liveViewOptions).then(() => wrapped); // TODO document Optionsreturn wrapped;};diff --git a/src/puppeteer_live_view_client.js b/src/puppeteer_live_view_client.jsindex a5b551cf0..d64954c25 100644--- a/src/puppeteer_live_view_client.js+++ b/src/puppeteer_live_view_client.js@@ -101,6 +101,18 @@ export const detailPage = ({ id, url, image, html }) => {`;};+export const errorPage = ({ id, url, error }) => {+ return `+<div id="${id}">+ <h3>There has been an error on page: ${url}</h3>+ <div class="error">+ <h4>${error.message}</h4>+ </div>+</div>++`;+};+/*** The wsHandler() function encapsulates the whole client-side* messaging and rendering logic. All commands that the clientdiff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.jsindex 068150775..1c5006fb7 100644--- a/src/puppeteer_live_view_server.js+++ b/src/puppeteer_live_view_server.js@@ -1,12 +1,10 @@import http from 'http';import EventEmitter from 'events';import Promise from 'bluebird';-import uuid from 'uuid/v4';import WebSocket from 'ws';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';-import { createTimeoutPromise } from './utils';-import { layout, indexPage, detailPage } from './puppeteer_live_view_client';+import { layout, indexPage, detailPage, errorPage } from './puppeteer_live_view_client';const LOCAL_IPV6 = '::';const LOCAL_IPV4 = '127.0.0.1';@@ -26,61 +24,22 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {super();this.browser = browser;this.pages = new Map(); // to track all pages and their creation order for listing- this._pageIds = new WeakMap(); // to avoid iteration over pages- this._loadedPages = new WeakSet(); // to just track loaded state+ this.pageIds = new WeakMap(); // to avoid iteration over pages+ this.loadedPages = new WeakSet(); // to just track loaded state+ this.pageIdCounter = 0;- checkParamOrThrow(opts.id, 'opts.id', 'Maybe String');+ checkParamOrThrow(opts.id, 'opts.id', 'String');checkParamOrThrow(opts.screenshotTimeout, 'opts.screenshotTimeout', 'Maybe Number');- this.id = opts.id || uuid();+ this.id = opts.id;this.screenshotTimeout = opts.screenshotTimeout || DEFAULT_SCREENSHOT_TIMEOUT;// since the page can be in any state when the user requests// a screenshot, we need to keep track of it ourselves- browser.on('targetcreated', (target) => {- if (target.type() === 'page') {- target.page()- .then((page) => {- const id = uuid();- this.pages.set(id, page);- this._pageIds.set(page, id);- this.emit('pagecreated', {- id,- browserId: this.id,- url: page.url(),- });- page.on('load', () => {- this._loadedPages.add(page); // page is loaded- });- // using "framenavigated" on main Frame since "targetchanged" does not seem reliable- page.on('framenavigated', (frame) => {- if (frame === page.mainFrame()) {- this._loadedPages.delete(page); // page will load after nav- this.emit('pagenavigated', {- id,- url: frame.url(),- });- }- });- })- .catch(err => log.error(err));- }- });+ browser.on('targetcreated', this._onTargetCreated.bind(this));// clean up resources after page close- browser.on('targetdestroyed', (target) => {- if (target.type() === 'page') {- target.page()- .then((page) => {- const id = this._pageIds.get(page);- this.pages.delete(id);- this.emit('pagedestroyed', {- id,- });- })- .catch(err => log.error(err));- }- });+ browser.on('targetdestroyed', this._onTargetDestroyed.bind(this));// clean up resources after browser close (in Server)- browser.on('disconnected', b => this.emit('disconnected', b));+ browser.on('disconnected', this._onBrowserDisconnected.bind(this));}/**@@ -95,7 +54,7 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {* @param {Page} page Puppeteer Page instance.*/startCapturing(page) {- const id = this._pageIds.get(page);+ const id = this.pageIds.get(page);if (!id) return;const capture = () => {log.debug(`Capturing page. ID: ${id}`);@@ -109,12 +68,16 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {});}).catch((err) => {- if (!err) return;+ this.emit(id, {+ id,+ url: page.url(),+ error: err,+ });log.error(err);});};// capture immediately for loaded pages- if (this._loadedPages.has(page)) capture();+ if (this.loadedPages.has(page)) capture();// setup recurrent capturingpage.on('load', () => {if (this.listenerCount(id)) capture();@@ -131,7 +94,7 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {* @param {Page} page*/stopCapturing(page) {- const id = this._pageIds.get(page);+ const id = this.pageIds.get(page);this.removeAllListeners(id);}@@ -164,16 +127,6 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {});};- // setup promise factories- const data = () => new Promise((resolve, reject) => {- const result = {};- const image = page.screenshot().then((s) => { result.image = s; });- const html = page.content().then((s) => { result.html = s; });- Promise.all([image, html])- .then(() => resolve(result))- .catch(reject);- });- const timeout = () => createTimeoutPromise(this.screenshotTimeout, 'Puppeteer Live View: Screenshot timed out.');const cleanup = () => {// replace the stolen close() method or call it,// if it should've been called externally@@ -185,7 +138,74 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {}};- return Promise.race([data(), timeout()]).finally(cleanup);+ return Promise.props({+ image: page.screenshot(),+ html: page.content(),+ })+ .timeout(this.screenshotTimeout, 'Puppeteer Live View: Screenshot timed out.')+ .finally(cleanup);+ }++ /**+ * Handler invoked whenever a Target is created in a Puppeteer's Browser.+ * @param {Target} target+ * @private+ */+ _onTargetCreated(target) {+ if (target.type() === 'page') {+ target.page()+ .then((page) => {+ const id = `${this.id}_PAGE_${++this.pageIdCounter}`;+ this.pages.set(id, page);+ this.pageIds.set(page, id);+ this.emit('pagecreated', {+ id,+ browserId: this.id,+ url: page.url(),+ });+ page.on('load', () => {+ this.loadedPages.add(page); // page is loaded+ });+ // using "framenavigated" on main Frame since "targetchanged" does not seem reliable+ page.on('framenavigated', (frame) => {+ if (frame === page.mainFrame()) {+ this.loadedPages.delete(page); // page will load after nav+ this.emit('pagenavigated', {+ id,+ url: frame.url(),+ });+ }+ });+ })+ .catch(err => log.error(err));+ }+ }++ /**+ * Handler invoked whenever a Target is destroyed in a Puppeteer's Browser.+ * @param {Target} target+ * @private+ */+ _onTargetDestroyed(target) {+ if (target.type() === 'page') {+ target.page()+ .then((page) => {+ const id = this.pageIds.get(page);+ this.pages.delete(id);+ this.emit('pagedestroyed', {+ id,+ });+ })+ .catch(err => log.error(err));+ }+ }+ /**+ * Handler invoked whenever a Browser is destroyed / disconnected in Puppeteer.+ * @param {Browser} browser+ * @private+ */+ _onBrowserDisconnected(browser) {+ this.emit('disconnected', browser);}}@@ -209,26 +229,26 @@ export class PuppeteerLiveViewBrowser extends EventEmitter {* @param {String} [opts.screenshotTimeout] Max time allowed for the screenshot taking process.* @returns {Promise<PuppeteerLiveViewServer>}*/-let defaultServer;-export const startPuppeteerLiveView = (browserPromise, opts = {}) => {+let liveViewServer;+export const registerBrowserForLiveView = (browserPromise, opts = {}) => {let serverPromise = Promise.resolve();- if (!defaultServer) {- defaultServer = new PuppeteerLiveViewServer(opts);- serverPromise = defaultServer.startServer();+ if (!liveViewServer) {+ liveViewServer = new PuppeteerLiveViewServer(opts);+ serverPromise = liveViewServer.startServer();}const browserOpts = {- id: opts.browserId,+ id: opts.browserId || `BROWSER_${++liveViewServer.browserIdCounter}`,screenshotTimeout: opts.screenshotTimeout,};browserPromise.then((browser) => {const lvb = new PuppeteerLiveViewBrowser(browser, browserOpts);- defaultServer.addBrowser(lvb);- return defaultServer;+ liveViewServer.addBrowser(lvb);+ return liveViewServer;});- return Promise.all([serverPromise, browserPromise]).then(() => defaultServer); // return PuppeteerLiveViewServer instance+ return Promise.all([serverPromise, browserPromise]).then(() => liveViewServer); // return PuppeteerLiveViewServer instance};/**@@ -269,8 +289,9 @@ export default class PuppeteerLiveViewServer extends EventEmitter {super();checkParamOrThrow(opts, 'opts', 'Object');checkParamOrThrow(opts.port, 'opts.port', 'Maybe String | Number');+ this.customPort = opts.port;this.browsers = new Set();- this.port = Number(opts.port) || 1234;+ this.browserIdCounter = 0;this.httpServer = null;}@@ -304,12 +325,18 @@ export default class PuppeteerLiveViewServer extends EventEmitter {*/startServer() {return new Promise((resolve, reject) => {+ const port = this.customPort == null ? process.env.APIFY_CONTAINER_PORT : this.customPort;+ if (port == null) {+ return reject(new Error('Neither options.port nor the environment variable APIFY_CONTAINER_PORT is set.' ++ 'LiveViewServer cannot be started.'));+ }+const server = http.createServer(this._httpRequestListener.bind(this));const wss = new WebSocket.Server({ server });wss.on('connection', this._wsRequestListener.bind(this));- server.listen(this.port, (err) => {+ server.listen(port, (err) => {if (err) reject(err);- log.info(`Live view server is listening on port ${this.port}.`);+ log.info(`Live view server is listening on port ${port}.`);this.httpServer = server;resolve();});@@ -371,35 +398,43 @@ export default class PuppeteerLiveViewServer extends EventEmitter {return [browser, page];};- log.debug('WebSocket connection to Puppeteer Live View established.');- ws.on('message', (msg) => {- try {- msg = JSON.parse(msg);- } catch (err) {- return ws.send(BAD_REQUEST);- }- const { command } = msg;-- if (command === 'renderIndex') {- sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });+ const COMMANDS = {+ renderIndex: () => sendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) }),// when the user selects a Page from the Index- } else if (command === 'renderPage') {+ renderPage: (msg) => {const { id } = msg.data || {};const [browser, page] = findPage(id);if (!page) return sendCommand(ws, 'error', NOT_FOUND);browser.startCapturing(page);- browser.on(id, pageData => sendCommand(ws, 'renderPage', { html: detailPage(pageData) }));+ browser.on(id, (pageData) => {+ if (pageData.error) sendCommand(ws, 'renderPage', { html: errorPage(pageData) });+ else sendCommand(ws, 'renderPage', { html: detailPage(pageData) });+ });+ },// when the user clicks on the back button- } else if (command === 'quitPage') {+ quitPage: (msg) => {const { id } = msg.data || {};const [browser, page] = findPage(id);if (!page) return; // no need to send errorbrowser.stopCapturing(page);- } else {- sendCommand(ws, 'error', BAD_REQUEST);+ },+ };++ log.debug('WebSocket connection to Puppeteer Live View established.');+ ws.on('message', (msg) => {+ try {+ msg = JSON.parse(msg);+ } catch (err) {+ return sendCommand(ws, 'error', BAD_REQUEST);}+ // validate command and send response+ const { command } = msg;+ const fn = COMMANDS[command];+ if (!fn || typeof fn !== 'function') return sendCommand(ws, 'error', BAD_REQUEST);+ fn(msg);});this._setupCommandHandles(ws);+ // send first Index after WS connection is establishedsendCommand(ws, 'renderIndex', { html: indexPage(this.browsers) });}diff --git a/test/puppeteer_live_view_server.js b/test/puppeteer_live_view_server.jsindex 9f43c4077..92dac1668 100644--- a/test/puppeteer_live_view_server.js+++ b/test/puppeteer_live_view_server.js@@ -1,7 +1,11 @@import http from 'http';import { expect, assert } from 'chai';import Promise from 'bluebird';+import puppeteer from 'puppeteer';import Apify from '../build/index';+import PuppeteerLiveViewServer, { PuppeteerLiveViewBrowser } from '../src/puppeteer_live_view_server';++const PORT = 1234;const httpGet = (port) => {return new Promise((resolve, reject) => {@@ -20,14 +24,76 @@ const httpGet = (port) => {});};+before(() => {+ process.env.APIFY_CONTAINER_PORT = PORT;+});+describe('Starting the PuppeteerLiveViewServer', () => {it('should start using Apify.launchPuppeteer()', () => {return Apify.launchPuppeteer({ liveView: true })- .then(() => httpGet(1234))+ .then(() => httpGet(PORT)).catch(err => assert.fail(err, 'Server response.')).then((body) => {expect(body.trim().substr(0, 15)).to.equal('<!doctype html>');});});+ it('should start using PuppeteerLiveViewServer.startServer()', () => {+ const server = new PuppeteerLiveViewServer({ port: 0 });+ return server.startServer()+ .then(() => {+ const { port } = server.httpServer.address();+ return httpGet(port);+ })+ .catch(err => assert.fail(err, 'Server response.'))+ .then((body) => {+ expect(body.trim().substr(0, 15)).to.equal('<!doctype html>');+ server.httpServer.close();+ });+ });+});++describe('Manipulate the PuppeteerLiveViewServer', () => {+ let server;++ beforeEach(() => {+ server = new PuppeteerLiveViewServer({ port: 0 });+ return server.startServer();+ });++ afterEach(() => {+ server.httpServer.close();+ server = null;+ });++ it('should add and remove browsers', () => {+ let createHandler = 0;+ server.on('browsercreated', () => {+ createHandler++;+ });++ let destroyHandler = 0;+ server.on('browserdestroyed', () => {+ destroyHandler++;+ });++ return puppeteer.launch()+ .then(browser => server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B1' })))+ .then(() => {+ expect(server.browsers.size).to.equal(1);+ expect(createHandler).to.equal(1);+ })+ .then(() => puppeteer.launch())+ .then(browser => server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B2' })))+ .then(() => {+ expect(server.browsers.size).to.equal(2);+ expect(createHandler).to.equal(2);+ })+ .then(() => server.deleteBrowser(server.browsers.values().next().value))+ .then(() => {+ expect(server.browsers.size).to.equal(1);+ expect(createHandler).to.equal(2);+ expect(destroyHandler).to.equal(1);+ });+ });});