Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
add documentation comments and do some cosmetic refactoring
Ondrej Urbanf18417bf528e
diff --git a/src/live_view.js b/src/live_view_browser.jssimilarity index 57%rename from src/live_view.jsrename to src/live_view_browser.jsindex 32f359406..7f3f9930a 100644--- a/src/live_view.js+++ b/src/live_view_browser.js@@ -1,30 +1,43 @@-import http from 'http';-import url from 'url';-import log from 'apify-shared/log';import Promise from 'bluebird';import { checkParamOrThrow } from 'apify-client/build/utils';-import { Router, dispatcher } from './live_view_router';+import { dispatcher } from './live_view_router';import { imgPage, errorPage } from './live_view_html';-class LiveViewBrowser {+/**+ * LiveViewBrowser encapsulates a Puppeteer's Browser instance and provides+ * an API to safely get screenshots of the Browser's Pages.+ * @param {Browser} browser A Puppeteer Browser instance.+ * @param {String} options.id A unique ID of the LiveViewBrowser.+ */+export default class LiveViewBrowser {constructor(browser, opts = {}) {this.browser = browser;+ // pages are stored in a WeakMap to be automatically garbage collected on close()this.pages = new WeakMap();checkParamOrThrow(opts.id, 'opts.id', 'Maybe String');this.id = opts.id;+ // since the page can be in any state when the user requests+ // a screenshot, we need to keep track of it ourselvesbrowser.on('targetcreated', (target) => {if (target.type() === 'page') {target.page().then((page) => {page.on('load', () => {- this.pages.set(page, true);+ this.pages.set(page, true); // page is loaded});});}});}++ /**+ * Handler that gets invoked by LiveViewRouter and sends an appropriate+ * response.+ * @param {http.IncomingMessage} req+ * @param {http.ServerResponse} res+ */routeHandler(req, res) {this.browser.pages().then((pages) => {@@ -40,28 +53,40 @@ class LiveViewBrowser {});}+ /**+ * The screenshot method simply takes a screenshot of the provided+ * Page and returns it as a promise. Unfortunately, nothing prevents+ * the Page from being closed while the screenshot is being taken,+ * which results into error. Therefore, the method prevents the page+ * from being closed by replacing its close method and handling the+ * page close itself once the screenshot has been taken.+ * @param {Page} page Puppeteer's Page+ * @returns {Promise<Buffer>} screenshot+ * @private+ */_screenshot(page) {- // replace page's close function to prevent close+ // replace page's close function to prevent a close// while the screenshot is being taken- let result;const { close } = page;let closed;let closeArgs;let closeResolve;page.close = (...args) => {+ if (!closed) closeArgs = args;closed = true;- closeArgs = args;return new Promise((resolve) => {closeResolve = resolve;});};+ // check if the page has been marked as loadedconst loaded = this.pages.get(page);- const timeoutPromise = new Promise(resolve => setTimeout(resolve, 500));+ const timeoutPromise = new Promise(resolve => setTimeout(resolve, 1000));// if page is already loaded, take a screenshot// otherwise, wait for it to load+ let result;if (loaded) {result = Promise.race([page.screenshot()], timeoutPromise);} else {@@ -91,50 +116,3 @@ class LiveViewBrowser {}}-export default class LiveViewServer {- constructor(opts = {}) {- checkParamOrThrow(opts, 'opts', 'Object');- checkParamOrThrow(opts.port, 'opts.port', 'Maybe String | Number');- this.browsers = [];- this.port = Number(opts.port) || 1234;- this.httpServer = null;- this.browserCounter = 0;- this.router = new Router();- }-- static start(browserPromise, opts = {}) {- if (!LiveViewServer.server) {- LiveViewServer.server = new LiveViewServer(opts);- LiveViewServer.server.startServer();- }- const lvs = LiveViewServer.server;- // TODO Ensure uniqueness of IDs.- const browserOpts = {- id: opts.browserId || `${++lvs.browserCounter}`,- };-- return browserPromise- .then((browser) => {- const lvb = new LiveViewBrowser(browser, browserOpts);- lvs.browsers.push(lvb);- lvs.router.addBrowser(lvb);- });- }-- startServer() {- const server = http.createServer(this._requestListener.bind(this));- server.listen(this.port, (err) => {- if (err) reject(err);- log.info(`Live view server is listening on port ${this.port}.`);- this.httpServer = server;- });- }-- _requestListener(req, res) {- const parsedUrl = url.parse(req.url, true);- const path = parsedUrl.pathname.replace(/^\/+|\/+$/g, '');- req.lvs = this;- this.router.handle(path, req, res);- }-}-diff --git a/src/live_view_html.js b/src/live_view_html.jsindex 279c1ca9b..3770b192b 100644--- a/src/live_view_html.js+++ b/src/live_view_html.js@@ -1,3 +1,9 @@+/**+ * Template for a basic layout of a HTML page.+ * @param {String} [opts.body] the body of the page+ * @param {Number} [opts.refresh] refresh time of the page+ * @returns {string} html+ */export const layout = (opts = {}) => `<!doctype html><html>@@ -13,6 +19,11 @@ ${opts.body || ''}</html>`;+/**+ * Generates a body for the rootPage. A list of browsers and their IDs.+ * @param {Array} browsers+ * @returns {string}+ */export const rootPage = (browsers) => {const body = `<h1>Available live view browsers:</h1>@@ -26,28 +37,46 @@ export const rootPage = (browsers) => {});};-export const encodeImg = (buffer) => {+/**+ * Returns an image encoded as a base64 string in an <img> tag.+ * @param {Buffer} imageBuffer+ * @returns {string}+ */+export const encodeImg = (imageBuffer) => {return `<div>- <img src="data:image/png;base64, ${buffer.toString('base64')}" alt="Page screenshot" />+ <img src="data:image/png;base64, ${imageBuffer.toString('base64')}" alt="Page screenshot" /></div>`;};--export const imgPage = (buffer) => {- const body = encodeImg(buffer);+/**+ * Returns a body of a page consisting of a serialized image.+ * @param {Buffer} imageBuffer+ * @returns {string}+ */+export const imgPage = (imageBuffer) => {+ const body = encodeImg(imageBuffer);return layout({body,refresh: 1,});};+/**+ * Returns a 404 response page.+ * @returns {string}+ */export const notFoundPage = () => {const body = '<p>This page does not exist.</p>';return layout({ body });};+/**+ * Returns an error page with an error message.+ * @param {String} message+ * @returns {string}+ */export const errorPage = (message) => {const body = `<p>Sorry. There was an error and Live View failed.</p>diff --git a/src/live_view_router.js b/src/live_view_router.jsindex c4c42608b..71fd78b14 100644--- a/src/live_view_router.js+++ b/src/live_view_router.js@@ -1,6 +1,12 @@import { checkParamOrThrow } from 'apify-client/build/utils';import { notFoundPage, rootPage } from './live_view_html';+/**+ * Initiates server response with predefined Content-Type and Content-Length headers.+ * @param {http.ServerResponse} res+ * @param {String | Buffer} body+ * @param {Number} [status]+ */export const dispatcher = (res, body, status) => {checkParamOrThrow(res, 'res', 'Object');checkParamOrThrow(body, 'body', 'Maybe String | Buffer');@@ -12,16 +18,28 @@ export const dispatcher = (res, body, status) => {res.end(body);};+/**+ * Individual route handlers that use HTML templates to serialize responses.+ * @param {http.IncomingMessage} req+ * @param {http.ServerResponse} res+ */const rootPageHandler = (req, res) => dispatcher(res, rootPage(req.lvs.browsers));-const notFoundHandler = (req, res) => dispatcher(res, notFoundPage());+const notFoundHandler = (req, res) => dispatcher(res, notFoundPage(), 404);-export class Router {+/**+ * LiveViewRouter class encapsulates routing logic of the LiveViewServer.+ */+export default class LiveViewRouter {constructor() {this.routes = {'': rootPageHandler,};}+ /**+ * Creates a route that points to the given LiveViewBrowser by its id.+ * @param {LiveViewBrowser} browser+ */addBrowser(browser) {checkParamOrThrow(browser.id, 'browser.id', 'String');checkParamOrThrow(browser.routeHandler, 'browser.routeHandler', 'Function');@@ -29,6 +47,13 @@ export class Router {this.routes[route] = browser.routeHandler.bind(browser);}+ /**+ * The main route handling function of LiveViewRouter. Attempts to invoke a specific route+ * handler. If none is found, invokes 404 handler.+ * @param {String} path+ * @param {http.IncomingMessage} req+ * @param {http.ServerResponse} res+ */handle(path, req, res) {const handler = this.routes[path];if (handler) return handler(req, res);diff --git a/src/live_view_server.js b/src/live_view_server.jsnew file mode 100644index 000000000..b2723cfb5--- /dev/null+++ b/src/live_view_server.js@@ -0,0 +1,82 @@+import url from 'url';+import http from 'http';+import log from 'apify-shared/log';+import { checkParamOrThrow } from 'apify-client/build/utils';+import LiveViewRouter from './live_view_router';+import LiveViewBrowser from './live_view_browser';++/**+ * Enables Live View monitoring of Act execution by spawning a web server that responds with a list+ * of available browsers at its root path. Once the user chooses a browser, LiveViewServer will+ * periodically serve screenshots of the selected browser's latest loaded page.+ * @param {Number} [options.port] Listening port of the LiveViewServer. Defaults to 1234.+ */+export default class LiveViewServer {+ constructor(opts = {}) {+ checkParamOrThrow(opts, 'opts', 'Object');+ checkParamOrThrow(opts.port, 'opts.port', 'Maybe String | Number');+ this.browsers = [];+ this.port = Number(opts.port) || 1234;+ this.httpServer = null;+ this.browserCounter = 0;+ this.router = new LiveViewRouter();+ }++ /**+ * The start method should cover most use cases of starting a LiveViewServer.+ * It creates a single instance of the server on its first invocation and subsequent+ * invocations only add more browsers to the current server instance. Individual browsers+ * are assigned unique IDs that will be used in displaying the browsers in an HTML index+ * available at the server's root route.+ *+ * The ID is customizable to better identify individual browsers.+ *+ * @param {Promise<Browser>} browserPromise A Promise for a Puppeteer's Browser.+ * @param {Object} [opts] Options to pass down to LiveViewServer constructor.+ * @param {String} [opts.browserId] Custom ID to be used with the browser instance.+ * @returns {Promise} The promise will resolve when the promise for the Puppeteer's Browser resolves.+ */+ static start(browserPromise, opts = {}) {+ if (!LiveViewServer.server) {+ LiveViewServer.server = new LiveViewServer(opts);+ LiveViewServer.server.startServer();+ }+ const lvs = LiveViewServer.server;+ // TODO Ensure uniqueness of IDs.+ const browserOpts = {+ id: opts.browserId || `${++lvs.browserCounter}`,+ };++ return browserPromise+ .then((browser) => {+ const lvb = new LiveViewBrowser(browser, browserOpts);+ lvs.browsers.push(lvb);+ lvs.router.addBrowser(lvb);+ });+ }++ /**+ * Starts an HTTP server on a preconfigured port or 1234.+ */+ startServer() {+ const server = http.createServer(this._requestListener.bind(this));+ server.listen(this.port, (err) => {+ if (err) reject(err);+ log.info(`Live view server is listening on port ${this.port}.`);+ this.httpServer = server;+ });+ }++ /**+ * Request handler function that delegates to LiveViewRouter.+ * @param {http.IncomingMessage} req+ * @param {http.ServerResponse} res+ * @private+ */+ _requestListener(req, res) {+ const parsedUrl = url.parse(req.url, true);+ const path = parsedUrl.pathname.replace(/^\/+|\/+$/g, '');+ req.lvs = this;+ this.router.handle(path, req, res);+ }+}diff --git a/src/puppeteer.js b/src/puppeteer.jsindex c6578b7ae..af78a10d1 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -5,7 +5,7 @@ import log from 'apify-shared/log';import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';import { newPromise, getTypicalChromeExecutablePath } from './utils';import { getApifyProxyUrl } from './actor';-import LiveViewServer from './live_view';+import LiveViewServer from './live_view_server';/* global process, require */@@ -163,7 +163,10 @@ export const launchPuppeteer = (opts = {}) => {optsCopy.args = optsCopy.args || [];optsCopy.args.push('--no-sandbox');if (optsCopy.headless === undefined || optsCopy.headless === null) {- optsCopy.headless = process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';+ // forcing headless with liveView, otherwise screenshots open a new browser window+ optsCopy.headless = optsCopy.liveView === true+ ? true+ : process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';}if (optsCopy.useChrome && (optsCopy.executablePath === undefined || optsCopy.executablePath === null)) {optsCopy.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();@@ -184,12 +187,6 @@ export const launchPuppeteer = (opts = {}) => {optsCopy.args.push(`--user-agent=${userAgent}`);}- // forcing headless with liveView, otherwise- // screenshots open a new browser window- if (optsCopy.liveView === true) {- optsCopy.headless = true;- }-let browserPromise;if (optsCopy.proxyUrl) {browserPromise = launchPuppeteerWithProxy(puppeteer, optsCopy);