Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Change live view snapshotting to time based

Ondra Urbana368384d6662
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index 0fe38ac1f..53667e0e4 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -22,6 +22,15 @@ const ensureDir = promisify(fs.ensureDir);
* that provides a simple frontend to viewing the captured snapshots. A snapshot consists of three
* pieces of information, the currently opened URL, the content of the page (HTML) and its screenshot.
*
+ * ```json
+ * {
+ * "pageUrl": "https://www.example.com",
+ * "htmlContent": "<html><body> ....",
+ * "screenshotIndex": 3,
+ * "createdAt": "2019-04-18T11:50:40.060Z"
+ * }
+ * ```
+ *
* `LiveViewServer` is useful when you want to be able to inspect the current browser status on demand.
* When no client is connected, the webserver consumes very low resources so it should have a close
* to zero impact on performance. Only once a client connects the server will start serving snapshots.
@@ -103,16 +112,7 @@ class LiveViewServer {
}
/**
- * Serves the snapshot to all connected clients.
- *
- * ```json
- * {
- * "pageUrl": "https://www.example.com",
- * "htmlContent": "<html><body> ....",
- * "screenshotIndex": 3
- * }
- * ```
- *
+ * Serves a snapshot to all connected clients.
* Screenshots are not served directly, only their index number
* which is used by client to retrieve the screenshot.
*
@@ -125,6 +125,13 @@ class LiveViewServer {
await this._pushSnapshot(snapshot);
}
+ /**
+ * @return {?Object}
+ */
+ getLastSnapshot() {
+ return this.lastSnapshot;
+ }
+
/**
* @return {boolean}
*/
@@ -165,7 +172,7 @@ class LiveViewServer {
this._deleteScreenshot(screenshotIndex - this.maxScreenshotFiles);
}
- const snapshot = { pageUrl, htmlContent, screenshotIndex };
+ const snapshot = { pageUrl, htmlContent, screenshotIndex, createdAt: new Date() };
this.lastSnapshot = snapshot;
return snapshot;
}
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index e36ee00a5..2ea8f83c8 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -15,6 +15,8 @@ import LiveViewServer from './live_view/live_view_server';
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
+const LIVE_VIEW_SNAPSHOT_MAX_AGE_MILLIS = 2000;
+const LIVE_VIEW_SNAPSHOT_TIMEOUT_MILLIS = 5000;
const DEFAULT_OPTIONS = {
reusePages: false,
@@ -657,7 +659,7 @@ class PuppeteerPool {
async recyclePage(page) {
if (this.liveViewServer && this.liveViewServer.hasClients()) {
await this._serveLiveView(page)
- .catch(err => log.exception(err, 'LiveView failed to be served.'));
+ .catch(err => log.exception(err, 'Live View failed to be served.'));
}
if (this.reusePages) {
page.removeAllListeners();
@@ -677,11 +679,13 @@ class PuppeteerPool {
async _serveLiveView(page) {
const browser = page.browser();
- const pages = await browser.pages();
+ const lastSnapshot = this.liveViewServer.getLastSnapshot();
- // We only serve the second page of the browser.
- // First page is about:blank and the others are disregarded.
- if (pages[1] !== page) return;
+ // Only make new snapshot after some time to prevent overloading.
+ if (lastSnapshot) {
+ const millisSinceLastSnapshot = Date.now() - lastSnapshot.createdAt;
+ if (millisSinceLastSnapshot < LIVE_VIEW_SNAPSHOT_MAX_AGE_MILLIS) return;
+ }
const instance = await this._findInstanceByBrowser(browser);
// Sometimes the browser gets killed and there's no instance.
@@ -689,7 +693,11 @@ class PuppeteerPool {
// Only take snapshots in the most recently opened browser.
if (instance.id !== this.browserCounter - 1) return;
- await this.liveViewServer.serve(page);
+ await addTimeoutToPromise(
+ this.liveViewServer.serve(page),
+ LIVE_VIEW_SNAPSHOT_TIMEOUT_MILLIS,
+ 'PuppeteerPool: Serving of Live View timed out.',
+ );
}
}
diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.js
index e3605308f..3a3cdc464 100644
--- a/test/live_view/live_view_server.js
+++ b/test/live_view/live_view_server.js
@@ -99,7 +99,8 @@ describe('LiveViewServer', () => {
it('should serve snapshot', async () => {
await lvs.serve(fakePage);
const snapshot = await new Promise(resolve => socket.on('snapshot', resolve));
- expect(snapshot).to.be.eql({ pageUrl: 'url', htmlContent: 'content', screenshotIndex: 0 });
+ expect(snapshot).to.include({ pageUrl: 'url', htmlContent: 'content', screenshotIndex: 0 });
+ expect(`"${snapshot.createdAt}"`).to.be.eql(JSON.stringify(new Date(snapshot.createdAt)));
});
it('should return screenshots', async () => {