Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix bugs in LiveView and do some type experiments
Ondra Urban0dadce62b22e
diff --git a/package.json b/package.jsonindex b99209b57..aa8eeac89 100644--- a/package.json+++ b/package.json@@ -74,6 +74,8 @@"@babel/core": "^7.0.0","@babel/preset-env": "^7.0.0","@babel/register": "^7.0.0",+ "@types/puppeteer": "^1.12.4",+ "@types/socket.io": "^2.1.2","basic-auth-parser": "^0.0.2","chai": "^4.2.0","chai-as-promised": "^7.1.1",diff --git a/src/index.js b/src/index.jsindex bfd21540a..cf4727dac 100644--- a/src/index.js+++ b/src/index.js@@ -49,7 +49,6 @@ logSystemInfo();* @module Apify*/module.exports = {- // Actormain,getEnv,call,@@ -61,55 +60,42 @@ module.exports = {client: apifyClient,addWebhook,- // Autoscaled poolAutoscaledPool,- // Basic crawlerBasicCrawler,- // Cheerio crawlerCheerioCrawler,- // DatasetpushData,openDataset,- // Eventsevents,initializeEvents,stopEvents,- // Key-value storegetValue,setValue,getInput,openKeyValueStore,- // PuppeteerlaunchPuppeteer,PuppeteerPool,PuppeteerCrawler,- // PseudoUrlPseudoUrl,- // RequestsRequest,RequestList,openRequestList,openRequestQueue,- // Settings rotatorSettingsRotator,- // Live ViewLiveViewServer,- // Webdriverbrowse,launchWebDriver,- // utilsutils: Object.assign(publicUtils, {puppeteer: puppeteerUtils,social: socialUtils,diff --git a/src/live_view/index.html b/src/live_view/index.htmlindex d102016b2..8195549fa 100644--- a/src/live_view/index.html+++ b/src/live_view/index.html@@ -144,7 +144,7 @@// Don't wait for the next screenshot to be made.$('#resumeButton').hide();$('#pauseButton').show();- showSnapshot(window.lastSnapshot);+ liveViewSocket.emit('getLastSnapshot');});liveViewSocket.on('disconnect', () => {$('#resumeButton').show();diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.jsindex d40c97fd8..2a7d2fd0b 100644--- a/src/live_view/live_view_server.js+++ b/src/live_view/live_view_server.js@@ -9,6 +9,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import { promisifyServerListen } from 'apify-shared/utilities';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import { addTimeoutToPromise } from '../utils';+import Snapshot from './snapshot';const writeFile = promisify(fs.writeFile);const unlink = promisify(fs.unlink);@@ -48,27 +49,28 @@ const DEFAULT_SCREENSHOT_DIR_PATH = path.resolve(LOCAL_STORAGE_DIR, 'live_view')** When running locally, it is often best to use a headful browser for debugging, since it provides* a better view into the browser, including DevTools, but `LiveViewServer` works too.- *- * @param {Object} [options]- * All `LiveViewServer` parameters are passed- * via an options object with the following keys:- * @param {string} [options.screenshotDirectoryPath]- * By default, the screenshots are saved to- * the `live_view` directory in the Apify local storage directory.- * Provide a different absolute path to change the settings.- * @param {number} [options.maxScreenshotFiles=10]- * Limits the number of screenshots stored- * by the server. This is to prevent using up too much disk space.- * @param {number} [options.snapshotTimeoutSecs=3]- * If a snapshot is not made within the timeout,- * its creation will be aborted. This is to prevent- * pages from being hung up by a stalled screenshot.- * @param {number} [options.maxSnapshotFrequencySecs=2]- * Use this parameter to further decrease the resource consumption- * of `LiveViewServer` by limiting the frequency at which it'll- * serve snapshots.*/class LiveViewServer {+ /**+ * @param {Object} [options]+ * All `LiveViewServer` parameters are passed+ * via an options object with the following keys:+ * @param {string} [options.screenshotDirectoryPath]+ * By default, the screenshots are saved to+ * the `live_view` directory in the Apify local storage directory.+ * Provide a different absolute path to change the settings.+ * @param {number} [options.maxScreenshotFiles=10]+ * Limits the number of screenshots stored+ * by the server. This is to prevent using up too much disk space.+ * @param {number} [options.snapshotTimeoutSecs=3]+ * If a snapshot is not made within the timeout,+ * its creation will be aborted. This is to prevent+ * pages from being hung up by a stalled screenshot.+ * @param {number} [options.maxSnapshotFrequencySecs=2]+ * Use this parameter to further decrease the resource consumption+ * of `LiveViewServer` by limiting the frequency at which it'll+ * serve snapshots.+ */constructor(options = {}) {const {screenshotDirectoryPath = DEFAULT_SCREENSHOT_DIR_PATH,@@ -86,7 +88,10 @@ class LiveViewServer {this.snapshotTimeoutMillis = snapshotTimeoutSecs * 1000;this.maxSnapshotFrequencyMillis = maxSnapshotFrequencySecs * 1000;- // Snapshot data+ /**+ * @type {?Snapshot}+ * @private+ */this.lastSnapshot = null;this.lastScreenshotIndex = 0;@@ -106,10 +111,15 @@ class LiveViewServer {* @return {Promise}*/async start() {- await ensureDir(this.screenshotDirectoryPath);- await promisifyServerListen(this.httpServer)(this.port);- log.info('Live view web server started', { publicUrl: this.liveViewUrl });this._isRunning = true;+ try {+ await ensureDir(this.screenshotDirectoryPath);+ await promisifyServerListen(this.httpServer)(this.port);+ log.info('Live view web server started', { publicUrl: this.liveViewUrl });+ } catch (err) {+ log.exception(err, 'Live view web server failed to start.');+ this._isRunning = false;+ }}/**@@ -146,8 +156,7 @@ class LiveViewServer {if (this.servingSnapshot) return;if (this.lastSnapshot) {- const lastSnapshotAgeMillis = Date.now() - this.lastSnapshot.createdAt;- if (lastSnapshotAgeMillis < this.maxSnapshotFrequencyMillis) return;+ if (this.lastSnapshot.age() < this.maxSnapshotFrequencyMillis) return;}try {@@ -182,12 +191,17 @@ class LiveViewServer {* Returns an absolute path to the screenshot with the given index.* @param {number} screenshotIndex* @return {string}- * @ignore+ * @private*/_getScreenshotPath(screenshotIndex) {return path.join(this.screenshotDirectoryPath, `${screenshotIndex}.jpeg`);}+ /**+ * @param {Page} page+ * @return {Promise<Snapshot>}+ * @private+ */async _makeSnapshot(page) {const pageUrl = page.url();log.info('Making live view snapshot.', { pageUrl });@@ -206,21 +220,25 @@ class LiveViewServer {this._deleteScreenshot(screenshotIndex - this.maxScreenshotFiles);}- const snapshot = { pageUrl, htmlContent, screenshotIndex, createdAt: new Date() };+ const snapshot = new Snapshot({ pageUrl, htmlContent, screenshotIndex });this.lastSnapshot = snapshot;return snapshot;}+ /**+ * @param {Snapshot} snapshot+ * @private+ */_pushSnapshot(snapshot) {// Send new snapshot to clients- log.debug('Sending live view snapshot', { snapshot });+ log.debug('Sending live view snapshot', { createdAt: snapshot.createdAt, pageUrl: snapshot.pageUrl });this.socketio.emit('snapshot', snapshot);}/*** Initiates an async delete and does not wait for it to complete.- * @param screenshotIndex- * @ignore+ * @param {number} screenshotIndex+ * @private*/_deleteScreenshot(screenshotIndex) {unlink(this._getScreenshotPath(screenshotIndex))@@ -242,12 +260,6 @@ class LiveViewServer {app.use('/', express.static(__dirname));- // Serves a JS file with the last snapshot, so the the client can immediately show something- app.get('/init-last-snapshot.js', (req, res) => {- res.set('Content-Type', 'text/javascript');- res.send(`window.lastSnapshot = ${JSON.stringify(this.lastSnapshot, null, 2)};`);- });-// Serves JPEG with the last screenshotapp.get('/screenshot/:index', (req, res) => {const screenshotIndex = req.params.index;@@ -263,13 +275,25 @@ class LiveViewServer {// Socket.io server used to send snapshots to clientthis.socketio = socketio(this.httpServer);- this.socketio.on('connection', (socket) => {- this.clientCount++;- log.info('Live view client connected', { clientId: socket.id });- socket.on('disconnect', (reason) => {- this.clientCount--;- log.info('Live view client disconnected', { clientId: socket.id, reason });- });+ this.socketio.on('connection', this._socketConnectionHandler.bind(this));+ }++ /**+ * @param {socketio.Socket} socket+ * @private+ */+ _socketConnectionHandler(socket) {+ this.clientCount++;+ log.info('Live view client connected', { clientId: socket.id });+ socket.on('disconnect', (reason) => {+ this.clientCount--;+ log.info('Live view client disconnected', { clientId: socket.id, reason });+ });+ socket.on('getLastSnapshot', () => {+ if (this.lastSnapshot) {+ log.debug('Sending live view snapshot', { createdAt: this.lastSnapshot.createdAt, pageUrl: this.lastSnapshot.pageUrl });+ this.socketio.emit('snapshot', this.lastSnapshot);+ }});}}diff --git a/src/live_view/snapshot.js b/src/live_view/snapshot.jsnew file mode 100644index 000000000..01d1d4c5b--- /dev/null+++ b/src/live_view/snapshot.js@@ -0,0 +1,29 @@+/**+ * Represents a live view snapshot to be served by {@link LiveViewServer}.+ * @property {string} pageUrl+ * @property {string} htmlContent+ * @property {number} screenshotIndex+ * @property {Date} createdAt+ * @ignore+ */+export default class Snapshot {+ /**+ * @param {Object} props+ * @param {string} props.pageUrl+ * @param {string} props.htmlContent+ * @param {number} props.screenshotIndex+ */+ constructor(props) {+ this.pageUrl = props.pageUrl;+ this.htmlContent = props.htmlContent;+ this.screenshotIndex = props.screenshotIndex;+ this.createdAt = new Date();+ }++ /**+ * @return {number}+ */+ age() {+ return Date.now() - this.createdAt;+ }+}diff --git a/src/utils.js b/src/utils.jsindex 96bfe20f5..7d68b97c6 100644--- a/src/utils.js+++ b/src/utils.js@@ -329,9 +329,11 @@ export const getTypicalChromeExecutablePath = () => {* Wraps the provided Promise with another one that rejects with the given errorMessage* after the given timeoutMillis, unless the original promise resolves or rejects earlier.*- * @param {Promise} promise+ * @template T+ * @param {Promise<T>} promise* @param {number} timeoutMillis* @param {string} errorMessage+ * @return {Promise<T>}* @ignore*/export const addTimeoutToPromise = (promise, timeoutMillis, errorMessage) => {