Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactor live view
Ondra Urbandf622c68d395
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.jsindex 8ee3390ef..1b243d717 100644--- a/src/live_view/live_view_server.js+++ b/src/live_view/live_view_server.js@@ -1,16 +1,26 @@-const http = require('http');-const fs = require('fs');-const express = require('express');-const socketio = require('socket.io');-const { promisifyServerListen } = require('apify-shared/utilities');-const { ENV_VARS, LOCAL_ENV_VARS } = require('apify-shared/consts');+import http from 'http';+import fs from 'fs';+import path from 'path';+import { promisify } from 'util';+import express from 'express';+import socketio from 'socket.io';+import log from 'apify-shared/log';+import { promisifyServerListen } from 'apify-shared/utilities';+import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';-const { utils: { log } } = Apify;+const DEFAULT_SCREENSHOT_DIRECTORY_PATH = path.resolve('live_view');+const MAX_SCREENSHOT_FILES = 10;-const DELETE_SCREENSHOT_FILE_TIMEOUT_MILLIS = 10 * 1000;-class LiveViewServer {- constructor() {+const writeFile = promisify(fs.writeFile);+const unlink = promisify(fs.unlink);++export default class LiveViewServer {+ constructor(options = {}) {+ const {+ screenshotDirectoryPath = DEFAULT_SCREENSHOT_DIRECTORY_PATH,+ } = options;+const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];this.port = parseInt(containerPort, 10);@@ -19,6 +29,7 @@ class LiveViewServer {+ `${ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);}this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];+ this.screenshotDirectoryPath = screenshotDirectoryPath;// Snapshot datathis.lastSnapshot = null;@@ -26,6 +37,7 @@ class LiveViewServer {this.screenshotIndexToFilePath = {};// Setup HTTP server and Express router+ this.isRunning = false;this.httpServer = http.createServer();this.app = express();@@ -74,45 +86,54 @@ class LiveViewServer {async start() {await promisifyServerListen(this.httpServer)(this.port);log.info('Live view web server started', { publicUrl: this.liveViewUrl });+ this.isRunning = true;}- /**- * To be called when the snapshot of screen and HTML content was saved.- */- async pushSnapshot(screenshotFilePath, htmlContent, pageUrl) {- log.info('LiveViewServer.pushSnapshot()', { pageUrl });+ async serve(page) {+ if (!this.hasClients()) return;+ const snapshot = await this._makeSnapshot(page);+ await this._pushSnapshot(snapshot);+ }- const prevScreenshotIndex = this.lastScreenshotIndex;+ hasClients() {+ return this.clientCount > 0;+ }- this.lastScreenshotIndex++;- this.lastSnapshot = {- pageUrl,- htmlContent,- screenshotIndex: this.lastScreenshotIndex,- };- this.screenshotIndexToFilePath[this.lastScreenshotIndex] = screenshotFilePath;+ getScreenshotPath(screenshotIndex) {+ return path.join(this.screenshotDirectoryPath, screenshotIndex);+ }- // Send new snapshot to clients- log.info('Sending live view snapshot', { snapshot: this.lastSnapshot });- this.socketio.emit('snapshot', this.lastSnapshot);-- // Delete screenshot after a while, maybe some client still wants to download it- const prevFilePath = this.screenshotIndexToFilePath[prevScreenshotIndex];- if (prevFilePath) {- setTimeout(() => {- delete this.screenshotIndexToFilePath[prevScreenshotIndex];- log.debug('Deleting screenshot', { path: prevFilePath });- fs.unlink(prevFilePath, (err) => {- if (err) log.exception(err, 'Cannot delete file', { path: prevFilePath });- });- }, DELETE_SCREENSHOT_FILE_TIMEOUT_MILLIS);+ async _makeSnapshot(page) {+ const pageUrl = page.url();+ log.info('Making live view snapshot.', { pageUrl });+ const [htmlContent, screenshot] = await Promise.all([+ page.content(),+ page.screenshot(),+ ]);++ const screenshotIndex = this.lastScreenshotIndex++;++ await writeFile(this.getScreenshotPath(screenshotIndex), screenshot);+ if (screenshotIndex > MAX_SCREENSHOT_FILES) {+ this._deleteScreenshot(MAX_SCREENSHOT_FILES - screenshotIndex);}- }- hasClients() {- return this.clientCount > 0;+ const snapshot = { pageUrl, htmlContent, screenshotIndex };+ this.lastSnapshot = snapshot;+ return snapshot;}-}+ /**+ * To be called when the snapshot of screen and HTML content was saved.+ */+ async _pushSnapshot(snapshot) {+ // Send new snapshot to clients+ log.info('Sending live view snapshot', { snapshot });+ this.socketio.emit('snapshot', snapshot);+ }-module.exports = LiveViewServer;+ _deleteScreenshot(screenshotIndex) { // eslint-disable-line class-methods-use-this+ unlink(this.getScreenshotPath(screenshotIndex))+ .catch(err => log.exception(err, 'Cannot delete live view screenshot.'));+ }+}diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex c428da77a..f3f81dcba 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -8,7 +8,8 @@ import LinkedList from 'apify-shared/linked_list';import rimraf from 'rimraf';import { checkParamOrThrow } from 'apify-client/build/utils';import { launchPuppeteer } from './puppeteer';-import { addTimeoutToPromise } from './utils';+import { addTimeoutToPromise, isAtHome } from './utils';+import LiveViewServer from './live_view/live_view_server';/* global process */@@ -30,6 +31,7 @@ const DEFAULT_OPTIONS = {launchPuppeteerFunction: launchPuppeteerOptions => launchPuppeteer(launchPuppeteerOptions),recycleDiskCache: false,+ liveView: false,};const mkdtempAsync = util.promisify(fs.mkdtemp);@@ -163,6 +165,7 @@ class PuppeteerPool {launchPuppeteerOptions,recycleDiskCache,proxyUrls,+ useLiveView,} = _.defaults({}, options, DEFAULT_OPTIONS);// Disabling due to memory leak.@@ -188,6 +191,7 @@ class PuppeteerPool {checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');// Enforce non-empty proxyUrls arrayif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');+ checkParamOrThrow(useLiveView, 'options.useLiveView', 'Maybe Boolean');// Config.this.reusePages = reusePages;@@ -197,6 +201,10 @@ class PuppeteerPool {this.killInstanceAfterMillis = killInstanceAfterMillis || killInstanceAfterSecs * 1000;this.recycledDiskCacheDirs = recycleDiskCache ? new LinkedList() : null;this.proxyUrls = proxyUrls ? _.shuffle(proxyUrls) : null;+ if (useLiveView && !isAtHome()) {+ log.warning('LiveView can only be used on the Apify Platform. Use headful mode for debugging locally.');+ }+ this.liveViewServer = useLiveView && isAtHome() ? new LiveViewServer() : null;this.launchPuppeteerFunction = async () => {// Do not modify passed launchPuppeteerOptions!const opts = _.clone(launchPuppeteerOptions) || {};@@ -630,19 +638,21 @@ class PuppeteerPool {}/**- * Recycles the page, which means that it will be enqueued for future reuse- * instead of spawning a new tab. This is done automatically unless the `reusePages`- * option of the `PuppeteerPool` constructor is set to false. You can also use this- * function manually to tell the pool that you no longer need this specific page- * and it can be reused in the future.+ * Closes the page, unless the `reuseTabs` option is set to true.+ * Then it would only flag the page for a future reuse, without actually closing it.*- * Using this function without setting the `reusePages` option to false causes- * this function to trigger a page.close().+ * NOTE: LiveView snapshotting is tied to this function. When `useLiveView` option+ * is set to true, a snapshot of the page will be taken just before closing the page+ * or flagging it for reuse.** @param {Page} page* @return {Promise}*/async recyclePage(page) {+ if (this.liveViewServer) {+ await this._serveLiveView(page)+ .catch(err => log.exception(err, 'LiveView failed to be served.'));+ }if (this.reusePages) {page.removeAllListeners();this.idlePages.push(page);@@ -658,6 +668,15 @@ class PuppeteerPool {}}}++ async _serveLiveView(page) {+ if (!this.liveViewServer.isRunning) await this.liveViewServer.start();+ const browser = page.browser();+ const instance = this._findInstanceByBrowser(browser);+ // Only take snapshots in the most recently opened browser.+ if (instance.id !== this.browserCounter) return;+ await this.liveViewServer.serve(page);+ }}export default PuppeteerPool;diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.jsnew file mode 100644index 000000000..56279f8f5--- /dev/null+++ b/test/live_view/live_view_server.js@@ -0,0 +1,15 @@+import path from 'path';+import { expect } from 'chai';+import { LOCAL_STORAGE_DIR } from '../_helper';+import LiveViewServer from '../../build/live_view/live_view_server';++const LOCAL_STORAGE_SUBDIR = path.join(LOCAL_STORAGE_DIR, 'live_view');++describe('LiveViewServer', () => {+ it('should construct', () => {+ const lvs = new LiveViewServer({+ screenshotDirectoryPath: LOCAL_STORAGE_SUBDIR,+ });+ expect(lvs).to.be.instanceOf(LiveViewServer);+ });+});